{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pima Indians Diabetes Data Set（皮马印第安人糖尿病数据集）进行分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#竞赛的评价指标为logloss\n",
    "from sklearn.metrics import log_loss  \n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据 & 数据探索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "2            8      183             64              0        0  23.3   \n",
       "3            1       89             66             23       94  28.1   \n",
       "4            0      137             40             35      168  43.1   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  \n",
       "2                     0.672   32        1  \n",
       "3                     0.167   21        0  \n",
       "4                     2.288   33        1  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# path to where the data lies\n",
    "dpath = './'\n",
    "train = pd.read_csv(dpath +\"diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "Pregnancies                 768 non-null int64\n",
      "Glucose                     768 non-null int64\n",
      "BloodPressure               768 non-null int64\n",
      "SkinThickness               768 non-null int64\n",
      "Insulin                     768 non-null int64\n",
      "BMI                         768 non-null float64\n",
      "DiabetesPedigreeFunction    768 non-null float64\n",
      "Age                         768 non-null int64\n",
      "Outcome                     768 non-null int64\n",
      "dtypes: float64(2), int64(7)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pregnancies                 0\n",
       "Glucose                     0\n",
       "BloodPressure               0\n",
       "SkinThickness               0\n",
       "Insulin                     0\n",
       "BMI                         0\n",
       "DiabetesPedigreeFunction    0\n",
       "Age                         0\n",
       "Outcome                     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.845052</td>\n",
       "      <td>120.894531</td>\n",
       "      <td>69.105469</td>\n",
       "      <td>20.536458</td>\n",
       "      <td>79.799479</td>\n",
       "      <td>31.992578</td>\n",
       "      <td>0.471876</td>\n",
       "      <td>33.240885</td>\n",
       "      <td>0.348958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.369578</td>\n",
       "      <td>31.972618</td>\n",
       "      <td>19.355807</td>\n",
       "      <td>15.952218</td>\n",
       "      <td>115.244002</td>\n",
       "      <td>7.884160</td>\n",
       "      <td>0.331329</td>\n",
       "      <td>11.760232</td>\n",
       "      <td>0.476951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>27.300000</td>\n",
       "      <td>0.243750</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>30.500000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>0.372500</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>140.250000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>127.250000</td>\n",
       "      <td>36.600000</td>\n",
       "      <td>0.626250</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>199.000000</td>\n",
       "      <td>122.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>846.000000</td>\n",
       "      <td>67.100000</td>\n",
       "      <td>2.420000</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Pregnancies     Glucose  BloodPressure  SkinThickness     Insulin  \\\n",
       "count   768.000000  768.000000     768.000000     768.000000  768.000000   \n",
       "mean      3.845052  120.894531      69.105469      20.536458   79.799479   \n",
       "std       3.369578   31.972618      19.355807      15.952218  115.244002   \n",
       "min       0.000000    0.000000       0.000000       0.000000    0.000000   \n",
       "25%       1.000000   99.000000      62.000000       0.000000    0.000000   \n",
       "50%       3.000000  117.000000      72.000000      23.000000   30.500000   \n",
       "75%       6.000000  140.250000      80.000000      32.000000  127.250000   \n",
       "max      17.000000  199.000000     122.000000      99.000000  846.000000   \n",
       "\n",
       "              BMI  DiabetesPedigreeFunction         Age     Outcome  \n",
       "count  768.000000                768.000000  768.000000  768.000000  \n",
       "mean    31.992578                  0.471876   33.240885    0.348958  \n",
       "std      7.884160                  0.331329   11.760232    0.476951  \n",
       "min      0.000000                  0.078000   21.000000    0.000000  \n",
       "25%     27.300000                  0.243750   24.000000    0.000000  \n",
       "50%     32.000000                  0.372500   29.000000    0.000000  \n",
       "75%     36.600000                  0.626250   41.000000    1.000000  \n",
       "max     67.100000                  2.420000   81.000000    1.000000  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 各属性的统计特性\n",
    "train.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 离群点检查"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGRxJREFUeJzt3Xu4XXV95/H3x0SKCIhAtEqiQY2X\n2CJgRNR5FNFasEiqjxeoiCjK1Md7tT46tkittladOjqiHVQEqYKKqNGi2ILISAU5yEWDWDOIcEQl\nIDevXPzOH2tluT2cy05yVjbZ5/16nv1kr/t3nbWzP3v91i1VhSRJAHcbdQGSpLsOQ0GS1DEUJEkd\nQ0GS1DEUJEkdQ0GS1DEUtNVKcnaSl8ww7EtJXtjTcl+W5KdJfp5klz6WsamS7JdkcqB7bZL9RliS\ntjKLR12AFoYkVwIvqar/2BLLq6oD+5hvkrsD/wzsW1WX9LGMjayngBVVtW664VX1yC1ckrZy7ilI\nG+e+wLbA2o2dMI2t4v9cEn8wLlBbxQdU4y3JS5OsS/KzJGuS3H9g2OOTXJDkpvbfx88wj/sluTTJ\n69vurmkpyRFJvp7k3UluSPKDJAcOTLt7knOS3JLkP5Icm+Rfp1nGQ4HvtZ03JjlrrhrbOt6e5Fzg\nl8CDppnvI9rxbmybew6eMv1LBrqPSPL19v05be9L2qas500z7yuTPLV9f7ckb0zy/5Jcn+RTSXZu\nhy1PUkmOTHIVcFaSbZP8azvuje263Xe6v7/Gh6GgkUqyP/CPwHOB+wE/BE5ph+0M/BvwPmAXmmab\nf5vajp9kOfA14P1V9e4ZFvVYmi/0XYF3Ah9JknbYJ4Bvtss4BnjBdDOoqv8CNjTH7FRV+w9Z4wuA\no4Ad2vUbrP3uwBeArwD3AV4JfDzJw2ZYj8F6nti+fVRVbV9Vn5xjklcBfw48Cbg/cANw7JRxngQ8\nAvhT4IXAvYBl7br9JfCruerS1s1Q0Kg9Hzi+qr5VVb8B3gQ8rv2i/zPg+1V1UlXdXlUnA5cDzxiY\nfiVwNvCWqjpuluX8sKo+VFV3ACfSBNB9kzwAeAxwdFXdWlVfB9ZsRP3D1HhCVa1th982Zfp9ge2B\nd7TLPwv4InDoRtQwrP8OvLmqJtu/9THAs6c0FR1TVb+oql8Bt9GEwUOq6o6qurCqbu6hLt2FGAoa\ntfsz8Ou5qn4OXA/sNnVY64ftsA2eD/wIOHWO5fxkYBm/bN9u3y7jZwP9AK7e1PpnqHG2+d0fuLqq\nfjvL9PPlgcBn26agG4HvAnfQHCfZYLDWk4AzgFOSXJPkne2ejcaYoaBRu4bmywqAJPek+XX6o6nD\nWg9oh21wDHAd8IkkizZh+T8Gdk6y3UC/ZRsx/TA1znYr4muAZVMOQA9O/wtgsLY/3IjaproaOLCq\ndhp4bVtV09ZaVbdV1d9V1Urg8cBBwOGbsXxtBQwFbUl3bw9ebngtpmnPf1GSPZP8AfAPwPlVdSVw\nOvDQJH+RZHF7IHUlTfPKBrcBzwHuCZy0sWf3VNUPgQngmCTbJHkcv9/0M5dhapzN+TRf/G9Icvf2\nmoJn0B5XAS4GnpVkuyQPAY6cMv1Pmebg9Qz+BXh7kgcCJFmSZPVMIyd5cpI/bsP2Zpq/9R1DLktb\nKUNBW9LpNAcqN7yOqaozgb8FPkPzq/3BwCEAVXU9za/T19E0Kb0BOKiqrhucaVXdCjyL5kDt8Ztw\n2ufzgce1y3gb8EngN8NMOGyNs0x/K3AwcCDNHs8HgMOr6vJ2lPcAt9J8+Z8IfHzKLI4BTmybhJ47\nx+LeS3O85CtJbgHOozkAP5M/pGmWu5mmqelrwJ3OytJ4iQ/ZkX5fkk8Cl1fVW0Zdi7SluaegBS/J\nY5I8uD2P/wBgNfC5UdcljYJXLUpNM8lpNAe4J4GXVdVFoy1JGg2bjyRJHZuPJEmdra75aNddd63l\ny5ePugxJ2qpceOGF11XVkrnG2+pCYfny5UxMTIy6DEnaqiSZeuX9tGw+kiR1DAVJUsdQkCR1DAVJ\nUsdQkCR1DAVJUsdQkCR1DAVJUqe3UEhyfJJrk3xnhuFJ8r4k65JcmmTvfuqY+bU5dtxx+nnuuOOm\nTzvMa3Om9TXa1+Z8NoaZdhTzVf+29Lbrc0/hBOCAWYYfCKxoX0cBH+yxlnl3yy0b139jx+ljWo3W\n5nw2Nne79zVf9W9Lb7veQqGqzgF+Nssoq4GPVeM8YKck9+urHknS3EZ5TGE3mgeJbzDZ9ruTJEcl\nmUgysX79+i1SnCQtRKMMhUzTb9qHO1TVcVW1qqpWLVky503+JEmbaJShMAksG+heClwzolokSYw2\nFNYAh7dnIe0L3FRVPx5hPRtlhx02rv/GjtPHtBqtzflsbO5272u+6t+W3na9PU8hycnAfsCuSSaB\ntwB3B6iqfwFOB54OrAN+Cbyojzr6etrozTePZlqNt74+G37mtl5betv1FgpVdegcwwt4eV/LlyRt\nPK9oliR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQ\nkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1\nDAVJUsdQkCR1DAVJUsdQkCR1eg2FJAck+V6SdUneOM3wByT5apKLklya5Ol91iNJml1voZBkEXAs\ncCCwEjg0ycopo/0N8Kmq2gs4BPhAX/VIkubW557CPsC6qrqiqm4FTgFWTxmngB3b9/cCrumxHknS\nHPoMhd2Aqwe6J9t+g44BDksyCZwOvHK6GSU5KslEkon169f3UaskiX5DIdP0qyndhwInVNVS4OnA\nSUnuVFNVHVdVq6pq1ZIlS3ooVZIE/YbCJLBsoHspd24eOhL4FEBVfQPYFti1x5okSbPoMxQuAFYk\n2T3JNjQHktdMGecq4CkASR5BEwq2D0nSiPQWClV1O/AK4AzguzRnGa1N8tYkB7ejvQ54aZJLgJOB\nI6pqahOTJGkLWdznzKvqdJoDyIP9jh54fxnwhD5rkCQNzyuaJUkdQ0GS1DEUJEkdQ0GS1DEUJEkd\nQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS\n1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1DEUJEkdQ0GS1Ok1\nFJIckOR7SdYleeMM4zw3yWVJ1ib5RJ/1SJJmt7ivGSdZBBwL/AkwCVyQZE1VXTYwzgrgTcATquqG\nJPfpqx5J0tz63FPYB1hXVVdU1a3AKcDqKeO8FDi2qm4AqKpre6xHkjSHoUIhyXZJ/jbJh9ruFUkO\nmmOy3YCrB7on236DHgo8NMm5Sc5LcsAMyz8qyUSSifXr1w9TsiRpEwy7p/BR4DfA49ruSeBtc0yT\nafrVlO7FwApgP+BQ4MNJdrrTRFXHVdWqqlq1ZMmSIUuWJG2sYUPhwVX1TuA2gKr6FdN/6Q+aBJYN\ndC8FrplmnM9X1W1V9QPgezQhIUkagWFD4dYk96D9pZ/kwTR7DrO5AFiRZPck2wCHAGumjPM54Mnt\nPHelaU66YsiaJEnzbNizj94CfBlYluTjwBOAI2aboKpuT/IK4AxgEXB8Va1N8lZgoqrWtMOeluQy\n4A7gr6vq+k1bFUnS5krV1Gb+GUZMdgH2pWk2Oq+qruuzsJmsWrWqJiYmRrFoSdpqJbmwqlbNNd7G\nnJK6G80v/m2AJyZ51qYWJ0m6axqq+SjJ8cAewFrgt23vAk7rqS5J0ggMe0xh36pa2WslkqSRG7b5\n6BtJDAVJGnPD7imcSBMMP6E5FTVAVdUevVUmSdrihg2F44EXAN/md8cUJEljZthQuKq9rkCSNMaG\nDYXL22cdfIGBK5mryrOPJGmMDBsK96AJg6cN9POUVEkaM0OFQlW9qO9CJEmjN+zzFJYm+WySa5P8\nNMlnkiztuzhJ0pa1Mc9TWAPcn+Z2F19o+0mSxsiwobCkqj5aVbe3rxMAn3YjSWNm2FC4LslhSRa1\nr8MAb3EtSWNm2FB4MfBc4CfAj4Fnt/0kSWNk2LOPrgIO7rkWSdKIDXv20YlJdhrovnd7O21J0hgZ\ntvloj6q6cUNHVd0A7NVPSZKkURk2FO6W5N4bOpLszPBXQ0uSthLDfrH/T+A/k5xKc3uL5wL/0FtV\nkqSRGPZA88eSTAD70zxL4VlVdVmvlUmStrhhn9F8UlW9ALhsmn6SpDEx7DGFRw52JFkEPHr+y5Ek\njdKsoZDkTUluAfZIcnOSW9rua4HPb5EKJUlbzKyhUFX/WFU7AO+qqh2raof2tUtVvWkL1ShJ2kKG\nPfvoS0meOLVnVZ0zz/VIkkZo2FD464H32wL7ABfSnI0kSRoTw56S+ozB7iTLgHf2UpEkaWSGPfto\nqkngj+azEEnS6A17ncL/prmSGZog2Qu4ZIjpDgDeCywCPlxV75hhvGcDnwYeU1UTw9QkSZp/wx5T\nuIzmi72Am4CTq+rc2SZor2U4FvgTmj2LC5KsmXoldJIdgFcB529k7ZKkeTZrKCRZTHOPoxcDV9Hc\n4mIZcHySb1bVbbNMvg+wrqquaOd1CrCagauiW39Pc3zi9Zu0BpKkeTPXMYV3ATsDu1fV3lW1F/Ag\nYCfg3XNMuxtw9UD3ZNuvk2QvYFlVfXG2GSU5KslEkon169fPsVhJ0qaaKxQOAl5aVbds6FFVNwMv\nA54+x7SZpl91A5O7Ae8BXjdXkVV1XFWtqqpVS5YsmWt0SdImmisUqqpqmp53MPAFP4NJmqamDZYC\n1wx070BzBtPZSa4E9gXWJFk1V9GSpH7MFQqXJTl8as8khwGXzzHtBcCKJLsn2QY4BFizYWBV3VRV\nu1bV8qpaDpwHHOzZR5I0OnOdffRy4LQkL6a5grmAxwD3AJ4524RVdXuSVwBn0Jy5dHxVrU3yVmCi\nqtbMNr0kacvLNK1Ddx4p2Z/m9tkB1lbVmX0XNpNVq1bVxIQ7E5K0MZJcWFVzNs8Pe5uLs4CzNrsq\nSdJd2qbe5kKSNIYMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUM\nBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlS\nx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHV6DYUkByT5XpJ1Sd44zfC/SnJZkkuTnJnkgX3WI0maXW+h\nkGQRcCxwILASODTJyimjXQSsqqo9gFOBd/ZVjyRpbn3uKewDrKuqK6rqVuAUYPXgCFX11ar6Zdt5\nHrC0x3okSXPoMxR2A64e6J5s+83kSOBL0w1IclSSiSQT69evn8cSJUmD+gyFTNOvph0xOQxYBbxr\nuuFVdVxVraqqVUuWLJnHEiVJgxb3OO9JYNlA91LgmqkjJXkq8GbgSVX1mx7rkSTNoc89hQuAFUl2\nT7INcAiwZnCEJHsB/wc4uKqu7bEWSdIQeguFqrodeAVwBvBd4FNVtTbJW5Mc3I72LmB74NNJLk6y\nZobZSZK2gD6bj6iq04HTp/Q7euD9U/tcviRp43hFsySpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqG\ngiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSp\nYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSps7jPmSc5AHgvsAj4\ncFW9Y8rwPwA+BjwauB54XlVdOb81zDysatPnu+OOcMstmz69NJON/VzO9FncYQe4+eb5qUkLR297\nCkkWAccCBwIrgUOTrJwy2pHADVX1EOA9wD/1Vc98MxB0VzHTZ9HPqDZFn81H+wDrquqKqroVOAVY\nPWWc1cCJ7ftTgacks/22lyT1qc9Q2A24eqB7su037ThVdTtwE7DL1BklOSrJRJKJ9evX91SuJKnP\nUJjuF//U1tJhxqGqjquqVVW1asmSJfNSnCTpzvoMhUlg2UD3UuCamcZJshi4F/CzHmuSJM2iz1C4\nAFiRZPck2wCHAGumjLMGeGH7/tnAWVWbc07QlrPDDqOuQGrM9Fn0M6pN0dspqVV1e5JXAGfQnJJ6\nfFWtTfJWYKKq1gAfAU5Kso5mD+GQ+a9jvufY8FQ/3VX4WdR86vU6hao6HTh9Sr+jB97/GnhOnzVI\nkobnFc2SpI6hIEnqGAqSpI6hIEnqGAqSpI6hIEnqGAqSpE62kguIO0nWAz/cxMl3Ba6bx3K2Fgtx\nvRfiOsPCXO+FuM6w8ev9wKqa8+ZxW10obI4kE1W1atR1bGkLcb0X4jrDwlzvhbjO0N9623wkSeoY\nCpKkzkILheNGXcCILMT1XojrDAtzvRfiOkNP672gjilIkma30PYUJEmzMBQkSZ0FEwpJDkjyvSTr\nkrxx1PX0IcmyJF9N8t0ka5O8uu2/c5J/T/L99t97j7rW+ZZkUZKLknyx7d49yfntOn+yffrfWEmy\nU5JTk1zebvPHLZBt/dr28/2dJCcn2XbctneS45Ncm+Q7A/2m3bZpvK/9brs0yd6bs+wFEQpJFgHH\nAgcCK4FDk6wcbVW9uB14XVU9AtgXeHm7nm8EzqyqFcCZbfe4eTXw3YHufwLe067zDcCRI6mqX+8F\nvlxVDwceRbP+Y72tk+wGvApYVVV/RPNUx0MYv+19AnDAlH4zbdsDgRXt6yjgg5uz4AURCsA+wLqq\nuqKqbgVOAVaPuKZ5V1U/rqpvte9vofmS2I1mXU9sRzsR+PPRVNiPJEuBPwM+3HYH2B84tR1lHNd5\nR+CJNI+0papuraobGfNt3VoM3CPJYmA74MeM2fauqnNoHlE8aKZtuxr4WDXOA3ZKcr9NXfZCCYXd\ngKsHuifbfmMryXJgL+B84L5V9WNoggO4z+gq68X/At4A/Lbt3gW4sapub7vHcXs/CFgPfLRtNvtw\nknsy5tu6qn4EvBu4iiYMbgIuZPy3N8y8bef1+22hhEKm6Te25+Im2R74DPCaqhrrx7onOQi4tqou\nHOw9zajjtr0XA3sDH6yqvYBfMGZNRdNp29FXA7sD9wfuSdN8MtW4be/ZzOvnfaGEwiSwbKB7KXDN\niGrpVZK70wTCx6vqtLb3TzfsTrb/Xjuq+nrwBODgJFfSNAvuT7PnsFPbvADjub0ngcmqOr/tPpUm\nJMZ5WwM8FfhBVa2vqtuA04DHM/7bG2betvP6/bZQQuECYEV7hsI2NAem1oy4pnnXtqV/BPhuVf3z\nwKA1wAvb9y8EPr+la+tLVb2pqpZW1XKa7XpWVT0f+Crw7Ha0sVpngKr6CXB1koe1vZ4CXMYYb+vW\nVcC+SbZrP+8b1nust3drpm27Bji8PQtpX+CmDc1Mm2LBXNGc5Ok0vyAXAcdX1dtHXNK8S/LfgP8L\nfJvfta//D5rjCp8CHkDzn+o5VTX1INZWL8l+wOur6qAkD6LZc9gZuAg4rKp+M8r65luSPWkOrm8D\nXAG8iOaH3lhv6yR/BzyP5my7i4CX0LShj832TnIysB/N7bF/CrwF+BzTbNs2HN9Pc7bSL4EXVdXE\nJi97oYSCJGluC6X5SJI0BENBktQxFCRJHUNBktQxFCRJHUNBYyHJm9s7Z16a5OIkj01yZZJdpxn3\nP+eY12fbeaxLclP7/uIkj59lngfPdvfdJMsH73gp3VUtnnsU6a4tyeOAg4C9q+o37Zf2jLdOrqrH\nzza/qnpmO9/9aK97GFjWTNOsYQwviNTC456CxsH9gOs2XKxUVddVVXeZf5J7JPlykpe23T9v/90v\nydkDzyT4eGb61v99r0zyrSTfTvLwdl5HJHl/+/6+7d7GJe3r90IoyYPam9g9pp3utLa+7yd558B4\nT0vyjXZZn27vaUWSdyS5rN0renfb7zlpni9wSZJzNuePqYXNUNA4+AqwLMl/JflAkicNDNse+ALw\niar60DTT7gW8huY5Gw+iuZfSXK6rqr1p7lv/+mmGvw/4WlU9iuZ+RGs3DGhvS/EZmqtOL2h770lz\nhe4fA89L87CkXYG/AZ7aLmsC+KskOwPPBB5ZVXsAb2vncTTwp+0yDx5iHaRpGQra6lXVz4FH0zxg\nZD3wySRHtIM/D3y0qj42w+TfrKrJqvotcDGwfIhFbrjR4IUzjL8/7YNOquqOqrqp7b+kreewqrp4\nYPwzq+qmqvo1zX18HkjzkKSVwLlJLqa5180DgZuBXwMfTvIsmtsaAJwLnNDuDS0aYh2kaXlMQWOh\nqu4AzgbOTvJtfnfjsHOBA5N8oqa/p8vg/XHuYLj/ExumGXb8DW6iue/9ExjYe5ihhgD/XlWHTp1J\nkn1obgR3CPAKYP+q+sskj6V52NDFSfasqus3ojYJcE9BYyDJw5KsGOi1J/DD9v3RwPXAB7ZgSWcC\nL2trW5TmKWkAt9I8LevwJH8xxzzOA56Q5CHtfLZL8tD2uMK9qup0mmavPdvhD66q86vqaOA6fv9W\nytLQDAWNg+2BEzccfKVpdjlmYPhrgG0HD+L27NXAk9s9lguBR24YUFW/oDlT6rVJZnwkbFWtB44A\nTm7X6Tzg4cAOwBfbfl8DXttO8q72wPd3gHOAS+Z9rbQgeJdUSVLHPQVJUsdQkCR1DAVJUsdQkCR1\nDAVJUsdQkCR1DAVJUuf/A9XUhh+cWWT5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x147a2978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAF3BJREFUeJzt3X2UZHV95/H3xxkQH2ZEYDTAjA7q\nGB2zJpiWBd2jiJoAq0ziIS6sz6LsejQmWRMPHDdK3DwYdWN0l5igooSoqGh0RJRsQOPRDUizKnEG\niCMitKg0ysP4EGDwu3/UnWvZdFdXP9wuuvv9OqcOdX/3d+/91o879en7UFWpKiRJArjPqAuQJN17\nGAqSpJahIElqGQqSpJahIElqGQqSpJahoGUryeeSvGyGeZ9O8qKOtvuKJN9L8sMkB3axjflKcnSS\nib7pHUmOHmFJWmbWjroArQ5JrgNeVlX/uBTbq6rjulhvkn2AvwCOrKqvdrGNOdZTwJaq2jXd/Kp6\n3BKXpGXOIwVpbh4K7AfsmOuC6VkW/+aS+AfjKrUsdlCtbElenmRXkh8k2Z7kkL55T0pyeZLbmv8+\naYZ1HJzkyiS/30y3p5aSvDjJF5K8NcktSb6Z5Li+ZQ9L8vkku5P8Y5Izk/zdNNt4NHBNM3lrkktm\nq7Gp40+SfBH4MfCIadb72Kbfrc3pnhOmLP+yvukXJ/lC8/zzTfNXm1NZ/2madV+X5BnN8/skOS3J\nN5J8P8mHkxzQzNucpJKckuR64JIk+yX5u6bvrc1re+h046+Vw1DQSCU5Bvgz4LnAwcC3gPOaeQcA\nnwLeARxI77TNp6aex0+yGfgn4H9X1Vtn2NS/p/eGfhDwZuA9SdLM+wDwpWYbZwAvmG4FVfWvwN7T\nMftX1TFD1vgC4FRgXfP6+mvfB/gk8A/AQ4DfBt6f5BdneB399TylefrLVfXAqvrQLIu8GvgN4KnA\nIcAtwJlT+jwVeCzw68CLgAcBm5rX9l+Bn8xWl5Y3Q0Gj9jzg7Kr6f1V1B3A6cFTzRv8fga9X1blV\ntaeqPghcDTy7b/mtwOeAN1TVWQO2862qeldV3Q2cQy+AHprkYcATgddX1Z1V9QVg+xzqH6bG91XV\njmb+XVOWPxJ4IPCmZvuXABcAJ8+hhmH9F+B1VTXRjPUZwIlTThWdUVU/qqqfAHfRC4NHVdXdVXVF\nVd3eQV26FzEUNGqH0PfXc1X9EPg+cOjUeY1vNfP2eh7wbeD8Wbbz3b5t/Lh5+sBmGz/oawO4Yb71\nz1DjoPUdAtxQVT8dsPxieTjw982poFuBq4C76V0n2au/1nOBi4DzktyY5M3NkY1WMENBo3YjvTcr\nAJI8gN5fp9+eOq/xsGbeXmcANwMfSLJmHtv/DnBAkvv3tW2aw/LD1Djoq4hvBDZNuQDdv/yPgP7a\nfmEOtU11A3BcVe3f99ivqqattaruqqo/qqqtwJOAZwEvXMD2tQwYClpK+zQXL/c+1tI7n/+SJL+S\n5L7AnwKXVdV1wIXAo5P85yRrmwupW+mdXtnrLuC3gAcA58717p6q+hYwDpyRZN8kR/Hzp35mM0yN\ng1xG743/tUn2aT5T8Gya6yrAV4DnJLl/kkcBp0xZ/ntMc/F6Bn8N/EmShwMk2ZBk20ydkzwtyb9r\nwvZ2emN995Db0jJlKGgpXUjvQuXexxlVdTHwh8BH6f3V/kjgJICq+j69v05fQ++U0muBZ1XVzf0r\nrao7gefQu1B79jxu+3wecFSzjT8GPgTcMcyCw9Y4YPk7gROA4+gd8fwV8MKqurrp8jbgTnpv/ucA\n75+yijOAc5pTQs+dZXNvp3e95B+S7AYupXcBfia/QO+03O30TjX9E3CPu7K0ssQf2ZF+XpIPAVdX\n1RtGXYu01DxS0KqX5IlJHtncx38ssA34+KjrkkbBTy1KvdMkH6N3gXsCeEVVfXm0JUmj4ekjSVLL\n00eSpNayO3100EEH1ebNm0ddhiQtK1dcccXNVbVhtn7LLhQ2b97M+Pj4qMuQpGUlydRP3k/L00eS\npJahIElqGQqSpJahIElqGQqSpJahIElqGQqSpJahIElqdRYKSc5OclOSr80wP0nekWRXkiuTPKGb\nOnyspsf69dPvB+vXD7/sTH1nWvdCDFPXYo3DoG0Nem2DxmMpx2q1Wuox7vJI4X3AsQPmHwdsaR6n\nAu/ssBatErt3z619uj4LWcdcdbHOmdY7aFvzmbd799KO1Wq11GPcWShU1eeBHwzosg342+q5FNg/\nycFd1SNJmt0orykcSu+HxPeaaNruIcmpScaTjE9OTi5JcZK0Go0yFDJN27Q/7lBVZ1XVWFWNbdgw\n65f8SZLmaZShMAFs6pveCNw4olokSYw2FLYDL2zuQjoSuK2qvjPCerQCrFs3t/bp+ixkHXPVxTpn\nWu+gbc1n3rp1SztWq9VSj3Fnv6eQ5IPA0cBBSSaANwD7AFTVXwMXAscDu4AfAy/pog5/bVQAt9/e\nTd+FWg7bWsoadU9LPf6dhUJVnTzL/AJe2dX2JUlz5yeaJUktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS\n1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIU\nJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEmtTkMhybFJrkmy\nK8lp08x/WJLPJvlykiuTHN9lPZKkwToLhSRrgDOB44CtwMlJtk7p9t+BD1fV4cBJwF91VY8kaXZd\nHikcAeyqqmur6k7gPGDblD4FrG+ePwi4scN6JEmz6DIUDgVu6JueaNr6nQE8P8kEcCHw29OtKMmp\nScaTjE9OTnZRqySJbkMh07TVlOmTgfdV1UbgeODcJPeoqarOqqqxqhrbsGFDB6VKkqDbUJgANvVN\nb+Sep4dOAT4MUFX/DOwHHNRhTZKkAboMhcuBLUkOS7IvvQvJ26f0uR54OkCSx9ILBc8PSdKIdBYK\nVbUHeBVwEXAVvbuMdiR5Y5ITmm6vAV6e5KvAB4EXV9XUU0ySpCWytsuVV9WF9C4g97e9vu/5TuDJ\nXdYgSRqen2iWJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUM\nBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlS\ny1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSq9NQSHJskmuS7Epy2gx9nptkZ5IdST7QZT2SpMHW\ndrXiJGuAM4FnAhPA5Um2V9XOvj5bgNOBJ1fVLUke0lU9kqTZdXmkcASwq6qurao7gfOAbVP6vBw4\ns6puAaiqmzqsR5I0i6FCIcn9k/xhknc101uSPGuWxQ4Fbuibnmja+j0aeHSSLya5NMmxM2z/1CTj\nScYnJyeHKVmSNA/DHim8F7gDOKqZngD+eJZlMk1bTZleC2wBjgZOBt6dZP97LFR1VlWNVdXYhg0b\nhixZkjRXw4bCI6vqzcBdAFX1E6Z/0+83AWzqm94I3DhNn09U1V1V9U3gGnohIUkagWFD4c4k96P5\nSz/JI+kdOQxyObAlyWFJ9gVOArZP6fNx4GnNOg+idzrp2iFrkiQtsmHvPnoD8BlgU5L3A08GXjxo\ngarak+RVwEXAGuDsqtqR5I3AeFVtb+b9WpKdwN3AH1TV9+f3UiRJC5Wqqaf5Z+iYHAgcSe+00aVV\ndXOXhc1kbGysxsfHR7FpSVq2klxRVWOz9ZvLLamH0vuLf1/gKUmeM9/iJEn3TkOdPkpyNvB4YAfw\n06a5gI91VJckaQSGvaZwZFVt7bQSSdLIDXv66J+TGAqStMINe6RwDr1g+C69W1EDVFU9vrPKJElL\nbthQOBt4AfAv/OyagiRphRk2FK5vPlcgSVrBhg2Fq5vfOvgkfZ9krirvPpKkFWTYULgfvTD4tb42\nb0mVpBVmqFCoqpd0XYgkafSG/T2FjUn+PslNSb6X5KNJNnZdnCRpac3l9xS2A4fQ+7qLTzZtkqQV\nZNhQ2FBV762qPc3jfYC/diNJK8ywoXBzkucnWdM8ng/4FdeStMIMGwovBZ4LfBf4DnBi0yZJWkGG\nvfvoeuCEjmuRJI3YsHcfnZNk/77pBzdfpy1JWkGGPX30+Kq6de9EVd0CHN5NSZKkURk2FO6T5MF7\nJ5IcwPCfhpYkLRPDvrH/T+D/Jjmf3tdbPBf4086qkiSNxLAXmv82yThwDL3fUnhOVe3stDJJ0pIb\n9jeaz62qFwA7p2mTJK0Qw15TeFz/RJI1wK8ufjmSpFEaGApJTk+yG3h8ktuT7G6mbwI+sSQVSpKW\nzMBQqKo/q6p1wFuqan1VrWseB1bV6UtUoyRpiQx799GnkzxlamNVfX6R65EkjdCwofAHfc/3A44A\nrqB3N5IkaYUY9pbUZ/dPJ9kEvLmTiiRJIzPs3UdTTQC/tJiFSJJGb9jPKfwvep9khl6QHA58dYjl\njgXeDqwB3l1Vb5qh34nAR4AnVtX4MDVJkhbfsNcUdtJ7Yy/gNuCDVfXFQQs0n2U4E3gmvSOLy5Ns\nn/pJ6CTrgFcDl82xdknSIhsYCknW0vuOo5cC19P7iotNwNlJvlRVdw1Y/AhgV1Vd26zrPGAbfZ+K\nbvwPetcnfn9er0CStGhmu6bwFuAA4LCqekJVHQ48AtgfeOssyx4K3NA3PdG0tZIcDmyqqgsGrSjJ\nqUnGk4xPTk7OsllJ0nzNFgrPAl5eVbv3NlTV7cArgONnWTbTtFU7M7kP8DbgNbMVWVVnVdVYVY1t\n2LBhtu6SpHmaLRSqqmqaxrvpe4OfwQS9U017bQRu7JteR+8Ops8luQ44EtieZGy2oiVJ3ZgtFHYm\neeHUxiTPB66eZdnLgS1JDkuyL3ASsH3vzKq6raoOqqrNVbUZuBQ4wbuPJGl0Zrv76JXAx5K8lN4n\nmAt4InA/4DcHLVhVe5K8CriI3p1LZ1fVjiRvBMaravug5SVJSy/TnB26Z6fkGHpfnx1gR1Vd3HVh\nMxkbG6vxcQ8mJGkuklxRVbOenh/2ay4uAS5ZcFWSpHu1+X7NhSRpBTIUJEktQ0GS1DIUJEktQ0GS\n1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIU\nJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1DIUJEktQ0GS1Oo0FJIc\nm+SaJLuSnDbN/P+WZGeSK5NcnOThXdYjSRqss1BIsgY4EzgO2AqcnGTrlG5fBsaq6vHA+cCbu6pH\nkjS7Lo8UjgB2VdW1VXUncB6wrb9DVX22qn7cTF4KbOywHknSLLoMhUOBG/qmJ5q2mZwCfHq6GUlO\nTTKeZHxycnIRS5Qk9esyFDJNW03bMXk+MAa8Zbr5VXVWVY1V1diGDRsWsURJUr+1Ha57AtjUN70R\nuHFqpyTPAF4HPLWq7uiwHknSLLo8Urgc2JLksCT7AicB2/s7JDkc+BvghKq6qcNaJElD6CwUqmoP\n8CrgIuAq4MNVtSPJG5Oc0HR7C/BA4CNJvpJk+wyrkyQtgS5PH1FVFwIXTml7fd/zZ3S5fUnS3PiJ\nZklSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlS\ny1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQJLUMBUlSy1CQ\nJLUMBUlSy1CQJLUMBUlSq9NQSHJskmuS7Epy2jTz75vkQ838y5JsXvwafKz2x/r10+8b69d3u62Z\n1j9TPcPWNdvy81n3Qta51Lp8DSthfBaqs1BIsgY4EzgO2AqcnGTrlG6nALdU1aOAtwF/3lU9Wr12\n755b+2Jta77bXej8+SzbxVh0pcvXsBLGZ6G6PFI4AthVVddW1Z3AecC2KX22Aec0z88Hnp4kHdYk\nSRqgy1A4FLihb3qiaZu2T1XtAW4DDpy6oiSnJhlPMj45OdlRuZKkLkNhur/4ax59qKqzqmqsqsY2\nbNiwKMVJku6py1CYADb1TW8EbpypT5K1wIOAH3RYkyRpgC5D4XJgS5LDkuwLnARsn9JnO/Ci5vmJ\nwCVVdY8jBWkh1q2bW/tibWu+213o/Pks28VYdKXL17ASxmeh1na14qrak+RVwEXAGuDsqtqR5I3A\neFVtB94DnJtkF70jhJMWv47FXqNWittvv3euv8u6un7NS8Hx6VZnoQBQVRcCF05pe33f838DfqvL\nGiRJw/MTzZKklqEgSWoZCpKklqEgSWoZCpKklqEgSWoZCpKkVpbbB4iTTALfmufiBwE3L2I5K43j\nM5jjM5jjM9iox+fhVTXrl8ctu1BYiCTjVTU26jrurRyfwRyfwRyfwZbL+Hj6SJLUMhQkSa3VFgpn\njbqAeznHZzDHZzDHZ7BlMT6r6pqCJGmw1XakIEkawFCQJLVWTSgkOTbJNUl2JTlt1PWMQpJNST6b\n5KokO5L8TtN+QJL/k+TrzX8f3LQnyTuaMbsyyRNG+wq6l2RNki8nuaCZPizJZc3YfKj5FUGS3LeZ\n3tXM3zzKupdCkv2TnJ/k6mYfOsp952eS/F7z7+prST6YZL/luP+silBIsgY4EzgO2AqcnGTraKsa\niT3Aa6rqscCRwCubcTgNuLiqtgAXN9PQG68tzeNU4J1LX/KS+x3gqr7pPwfe1ozNLcApTfspwC1V\n9SjgbU2/le7twGeq6jHAL9MbJ/cdIMmhwKuBsar6JXq/NnkSy3H/qaoV/wCOAi7qmz4dOH3UdY36\nAXwCeCZwDXBw03YwcE3z/G+Ak/v6t/1W4gPYSO+N7RjgAiD0PoG6dup+RO9nZo9qnq9t+mXUr6HD\nsVkPfHPqa3TfaV/focANwAHN/nAB8OvLcf9ZFUcK/Ox/2F4TTduq1RyuHg5cBjy0qr4D0Pz3IU23\n1TZufwm8FvhpM30gcGtV7Wmm+19/OzbN/Nua/ivVI4BJ4L3N6bV3J3kA7jsAVNW3gbcC1wPfobc/\nXMEy3H9WSyhkmrZVey9ukgcCHwV+t6oG/VT5qhm3JM8CbqqqK/qbp+laQ8xbidYCTwDeWVWHAz/i\nZ6eKprOqxqe5lrINOAw4BHgAvVNoU93r95/VEgoTwKa+6Y3AjSOqZaSS7EMvEN5fVR9rmr+X5OBm\n/sHATU37ahq3JwMnJLkOOI/eKaS/BPZPsrbp0//627Fp5j8I+MFSFrzEJoCJqrqsmT6fXki47/Q8\nA/hmVU1W1V3Ax4AnsQz3n9USCpcDW5o7AfaldwFo+4hrWnJJArwHuKqq/qJv1nbgRc3zF9G71rC3\n/YXNnSRHArftPVWw0lTV6VW1sao209s/Lqmq5wGfBU5suk0dm71jdmLT/17xl14Xquq7wA1JfrFp\nejqwE/edva4Hjkxy/+bf2d7xWX77z6gvaizhhaDjgX8FvgG8btT1jGgM/gO9Q9Qrga80j+Ppncu8\nGPh6898Dmv6hd9fWN4B/oXdnxchfxxKM09HABc3zRwBfAnYBHwHu27Tv10zvauY/YtR1L8G4/Aow\n3uw/Hwce7L7zc+PzR8DVwNeAc4H7Lsf9x6+5kCS1VsvpI0nSEAwFSVLLUJAktQwFSVLLUJAktQwF\nqU+SHy7y+jYn+VrzfCzJOxZz/dJiWzt7F0mLoarG6d3nL91reaQgTSPJ0Uk+1/f7Ae9vPqlKkjcl\n2dn8TsBbm7b3JTmxb/l7HHE069z7Ow1nJDm72ca1SV69VK9NGsQjBWlmhwOPo/d9NV8EnpxkJ/Cb\nwGOqqpLsv4D1PwZ4GrAOuCbJO6v3vTnSyHikIM3sS1U1UVU/pfeVIJuB24F/A96d5DnAjxew/k9V\n1R1VdTO9L5J76EILlhbKUJBmdkff87vp/VjKHuAIet80+xvAZ5r5e2j+PTWnmfadz/oXWrC0UIaC\nNAfNb1E8qKouBH6X3pfEAVwH/GrzfBuwz9JXJy2cf5lIc7MO+ESS/eh9E+jvNe3vatq/RO/bQn80\novqkBfFbUiVJLU8fSZJahoIkqWUoSJJahoIkqWUoSJJahoIkqWUoSJJa/x/USpcTy9bvgAAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1474fd30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEWCAYAAABi5jCmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAF3dJREFUeJzt3XuUpHV95/H3xxkQ0EGEaQ0wo4M6\nGidZFG0J6B5FvOzAIux6vMB6QUXZ9WguZzUeOG6UaBKNmhizIckSRZEooHgbDYoJeDm6gjQqxuES\nJ4jSGYUBQYiycvG7fzxPaz1FX6qbqa7unvfrnDpdz63q0z3P9Kfr9zz1VKoKSZKm3G/UASRJS4vF\nIEnqsBgkSR0WgySpw2KQJHVYDJKkDotBy1aSLyZ55QzLPpvkxCE976uT3JDk35PsN4znWKgkRySZ\n7JnemuSIEUbSMrR61AG0a0hyHfDKqvqnxXi+qjpqGI+bZDfgz4HDquqKYTzHPPMUsLGqtk23vKp+\nY5EjaQXwFYM0Pw8F9gC2znfDNJbF/7kk/tG4C1sWO6lWtiSvSrItyY+TbElyQM+yJye5LMlP2q9P\nnuEx9k/y7SSvb6d/OcyU5GVJvpLkXUluSfK9JEf1bHtQki8nuT3JPyU5PcnfT/McjwauaSdvTXLx\nXBnbHH+c5KvAz4BHTPO4j23Xu7Ud+jm2b/tX9ky/LMlX2vtfbmdf0Q5rvXCax74uyTPb+/dLckqS\nf01yc5KPJNm3XbYhSSU5KckPgIuT7JHk79t1b22/t4dO9/PXymIxaKSSHAm8DXgBsD/wfeDcdtm+\nwD8AfwnsRzOE8w/94/pJNgBfAv6qqt41w1P9Fs0v9bXAO4D3JUm77MPA19vnOA14yXQPUFX/AkwN\nzexTVUcOmPElwMnAmvb7682+G/Bp4PPAQ4DfBj6U5DEzfB+9eZ7a3n1cVT2wqs6bY5PfAf4L8DTg\nAOAW4PS+dZ4GPBb4T8CJwIOA9e339j+AO+bKpeXPYtCovQg4s6q+UVU/B04FDm9/2f9n4LtVdXZV\n3V1V5wBXA8/p2X4T8EXgzVV1xizP8/2q+ruqugc4i6aEHprkYcCTgDdV1Z1V9RVgyzzyD5LxA1W1\ntV1+V9/2hwEPBN7ePv/FwGeAE+aRYVD/HXhjVU22P+vTgOf1DRudVlU/rao7gLtoCuFRVXVPVV1e\nVbcNIZeWGItBo3YAPX9FV9W/AzcDB/Yva32/XTblRcC/AefP8Tw/6nmOn7V3H9g+x4975gFcv9D8\nM2Sc7fEOAK6vql/Msv3O8nDgE+2w0K3AVcA9NMdNpvRmPRu4EDg3yfYk72hf4WiFsxg0attpfmEB\nkOQBNH+l/lv/stbD2mVTTgNuAj6cZNUCnv+HwL5J9uqZt34e2w+ScbZLGG8H1vcdlO7d/qdAb7Zf\nm0e2ftcDR1XVPj23Papq2qxVdVdV/WFVbQKeDBwDvPQ+PL+WCYtBi2m39oDm1G01zfj+y5M8Psn9\ngT8BLq2q64ALgEcn+W9JVrcHVzfRDLVMuQt4PvAA4Oz5nvVTVd8HJoDTkuye5HC6w0BzGSTjbC6l\n+eX/hiS7te85eA7tcRbgW8Bzk+yV5FHASX3b38A0B7Rn8LfAHyd5OECSsSTHzbRykqcn+Q9t4d5G\n87O+Z8Dn0jJmMWgxXUBz8HLqdlpVXQT8AfAxmr/eHwkcD1BVN9P8lfo6muGlNwDHVNVNvQ9aVXcC\nz6U5eHvmAk4JfRFwePscfwScB/x8kA0HzTjL9ncCxwJH0bzy+WvgpVV1dbvKu4E7aQrgLOBDfQ9x\nGnBWOzz0gjme7j00x08+n+R24BKag/Iz+TWaIbrbaIadvgTc62wtrTzxg3qkriTnAVdX1ZtHnUUa\nBV8xaJeX5ElJHtme578ZOA745KhzSaPiuxulZsjk4zQHvSeBV1fVN0cbSRodh5IkSR0OJUmSOpbd\nUNLatWtrw4YNo44hScvK5ZdfflNVjQ2y7rIrhg0bNjAxMTHqGJK0rCTpf4f+jBxKkiR1WAySpA6L\nQZLUYTFIkjosBklSh8UgSeqwGCRJHRaDJKljaMWQ5MwkNyb5zgzLk+Qvk2xL8u0kTxhOjplvGr29\n957938jbyr/tvffC94upbWdaf7rHvi/75UIf774YRZZhvmL4ALB5luVHARvb28nA3wwxi5ao228f\ndQKN2nT7wKD7xdR6M62/0P1rZz/efTGKLEMrhqr6MvDjWVY5DvhgNS4B9kmy/7DySJIGM8pjDAfS\nfDj5lMl23r0kOTnJRJKJHTt2LEo4SdpVjbIYMs28aT8coqrOqKrxqhofGxvo4oCSpAUaZTFMAut7\nptcB20eURZLUGmUxbAFe2p6ddBjwk6r64QjzaATWrBl1Ao3adPvAoPvF1Hozrb/Q/WtnP959MYos\nQ/s8hiTnAEcAa5NMAm8GdgOoqr8FLgCOBrYBPwNePowcfnLp0nbbbaNOoKVovvvFzt6PltJ+OYos\nQyuGqjphjuUFvGZYzy9JWhjf+SxJ6rAYJEkdFoMkqcNikCR1WAySpA6LQZLUYTFIkjosBklSh8Ug\nSeqwGCRJHRaDJKnDYpAkdVgMkqQOi0GS1GExSJI6LAZJUofFIEnqsBgkSR0WgySpw2KQJHVYDJKk\nDotBktRhMUiSOiwGSVKHxSBJ6rAYJEkdFoMkqcNikCR1WAySpI6hFkOSzUmuSbItySnTLH9Yki8k\n+WaSbyc5eph5JElzG1oxJFkFnA4cBWwCTkiyqW+1/wV8pKoOAY4H/npYeSRJgxnmK4ZDgW1VdW1V\n3QmcCxzXt04Be7f3HwRsH2IeSdIAhlkMBwLX90xPtvN6nQa8OMkkcAHw29M9UJKTk0wkmdixY8cw\nskqSWsMshkwzr/qmTwA+UFXrgKOBs5PcK1NVnVFV41U1PjY2NoSokqQpwyyGSWB9z/Q67j1UdBLw\nEYCq+hqwB7B2iJkkSXMYZjFcBmxMclCS3WkOLm/pW+cHwDMAkjyWphgcK5KkERpaMVTV3cBrgQuB\nq2jOPtqa5C1Jjm1Xex3wqiRXAOcAL6uq/uEmSdIiWj3MB6+qC2gOKvfOe1PP/SuBpwwzgyRpfnzn\nsySpw2KQJHVYDJKkDotBktRhMUiSOiwGSVKHxSBJ6rAYJEkdFoMkqcNikCR1WAySpA6LQZLUYTFI\nkjosBklSh8UgSeqwGCRJHRaDJKnDYpAkdVgMkqQOi0GS1GExSJI6LAZJUofFIEnqsBgkSR0WgySp\nw2KQJHVYDJKkDotBktRhMUiSOiwGSVLHUIshyeYk1yTZluSUGdZ5QZIrk2xN8uFh5pEkzW31sB44\nySrgdOBZwCRwWZItVXVlzzobgVOBp1TVLUkeMqw8kqTBDPMVw6HAtqq6tqruBM4Fjutb51XA6VV1\nC0BV3TjEPJKkAQxUDEn2SvIHSf6und6Y5Jg5NjsQuL5nerKd1+vRwKOTfDXJJUk2z/D8JyeZSDKx\nY8eOQSJLkhZo0FcM7wd+DhzeTk8CfzTHNplmXvVNrwY2AkcAJwDvTbLPvTaqOqOqxqtqfGxsbMDI\nkqSFGLQYHllV7wDuAqiqO5j+F3+vSWB9z/Q6YPs063yqqu6qqu8B19AUhSRpRAYthjuT7En7F3+S\nR9K8gpjNZcDGJAcl2R04HtjSt84ngae3j7mWZmjp2gEzSZKGYNCzkt4MfA5Yn+RDwFOAl822QVXd\nneS1wIXAKuDMqtqa5C3ARFVtaZc9O8mVwD3A71fVzQv7ViRJO0Oq+of9Z1gx2Q84jGYI6ZKqummY\nwWYyPj5eExMTo3hqSVq2klxeVeODrDuf01UPpPnLf3fgqUmeu5BwkqSlbaChpCRnAgcDW4FftLML\n+PiQckmSRmTQYwyHVdWmoSaRJC0Jgw4lfS2JxSBJu4BBXzGcRVMOP6I5TTVAVdXBQ0smSRqJQYvh\nTOAlwD/zq2MMkqQVaNBi+EH7vgNJ0go3aDFc3X5WwqfpecdzVXlWkiStMIMWw540hfDsnnmeripJ\nK9BAxVBVLx92EEnS0jDo5zGsS/KJJDcmuSHJx5KsG3Y4SdLim8/nMWwBDqC5NMan23mSpBVm0GIY\nq6r3V9Xd7e0DgJ+YI0kr0KDFcFOSFydZ1d5eDHh5bElagQYthlcALwB+BPwQeF47T5K0wgx6VtIP\ngGOHnEWStAQMelbSWUn26Zl+cHspbknSCjPoUNLBVXXr1ERV3QIcMpxIkqRRGrQY7pfkwVMTSfZl\n8HdNS5KWkUF/uf8Z8H+TnE9zKYwXAH8ytFSSpJEZ9ODzB5NMAEfSfBbDc6vqyqEmkySNxKCf+Xx2\nVb0EuHKaeZKkFWTQYwy/0TuRZBXwxJ0fR5I0arMWQ5JTk9wOHJzktiS3t9M3Ap9alISSpEU1azFU\n1duqag3wzqrau6rWtLf9qurURcooSVpEg56V9NkkT+2fWVVf3sl5JEkjNmgx/H7P/T2AQ4HLac5S\nkiStIIOervqc3ukk64F3DCWRJGmkBj0rqd8k8Js7M4gkaWkY9H0M/5vmHc/QlMkhwBUDbLcZeA+w\nCnhvVb19hvWeB3wUeFJVTQySSZI0HIMeY7iS5pd7AT8Bzqmqr862Qfteh9OBZ9G8wrgsyZb+d0wn\nWQP8DnDpPLNLkoZg1mJIsprmmkivAH5AczmM9cCZSb5eVXfNsvmhwLaqurZ9rHOB4+h593TrrTTH\nK16/oO9AkrRTzXWM4Z3AvsBBVfWEqjoEeASwD/CuObY9ELi+Z3qynfdLSQ4B1lfVZ2Z7oCQnJ5lI\nMrFjx445nlaSdF/MVQzHAK+qqtunZlTVbcCrgaPn2DbTzKtfLkzuB7wbeN1cIavqjKoar6rxsbGx\nuVaXJN0HcxVDVVVNM/Meen7Jz2CSZthpyjpge8/0Gpozm76Y5DrgMGBLkvG5QkuShmeuYrgyyUv7\nZyZ5MXD1HNteBmxMclCS3YHjgS1TC6vqJ1W1tqo2VNUG4BLgWM9KkqTRmuuspNcAH0/yCpp3Ohfw\nJGBP4L/OtmFV3Z3ktcCFNGc0nVlVW5O8BZioqi2zbS9JGo1MM1J075WSI2kuvR1ga1VdNOxgMxkf\nH6+JCV9USNJ8JLm8qgYaqh/0khgXAxffp1SSpGVhoZfEkCStUBaDJKnDYpAkdVgMkqQOi0GS1GEx\nSJI6LAZJUofFIEnqsBgkSR0WgySpw2KQJHVYDJKkDotBktRhMUiSOiwGSVKHxSBJ6rAYJEkdFoMk\nqcNikCR1WAySpA6LQZLUYTFIkjosBklSh8UgSeqwGCRJHRaDJKnDYpAkdVgMkqQOi0GS1DHUYkiy\nOck1SbYlOWWa5f8zyZVJvp3koiQPH2YeSdLchlYMSVYBpwNHAZuAE5Js6lvtm8B4VR0MnA+8Y1h5\nJEmDGeYrhkOBbVV1bVXdCZwLHNe7QlV9oap+1k5eAqwbYh5J0gCGWQwHAtf3TE+282ZyEvDZ6RYk\nOTnJRJKJHTt27MSIkqR+wyyGTDOvpl0xeTEwDrxzuuVVdUZVjVfV+NjY2E6MKEnqt3qIjz0JrO+Z\nXgds718pyTOBNwJPq6qfDzGPJGkAw3zFcBmwMclBSXYHjge29K6Q5BDg/wDHVtWNQ8wiSRrQ0Iqh\nqu4GXgtcCFwFfKSqtiZ5S5Jj29XeCTwQ+GiSbyXZMsPDSZIWyTCHkqiqC4AL+ua9qef+M4f5/JKk\n+fOdz5KkDotBktRhMUiSOiwGSVKHxSBJ6rAYJEkdFoMkqcNikCR1WAySpA6LQZLUYTFIkjosBklS\nh8UgSeqwGCRJHRaDJKnDYpAkdVgMkqQOi0GS1GExSJI6LAZJUofFIEnqsBgkSR0WgySpw2KQJHVY\nDJKkDotBktRhMUiSOiwGSVKHxSBJ6lg9zAdPshl4D7AKeG9Vvb1v+f2BDwJPBG4GXlhV1+3cDDMv\nq9qZz7TyzfazlKazZg3cdtuoU/zK3nvD7bffe/5SyzlqQ3vFkGQVcDpwFLAJOCHJpr7VTgJuqapH\nAe8G/nRYeSQtvul+CY/STHmWWs5RG+ZQ0qHAtqq6tqruBM4Fjutb5zjgrPb++cAzEv8ulaRRGmYx\nHAhc3zM92c6bdp2quhv4CbBf/wMlOTnJRJKJHTt2DCmuJAmGWwzT/eXfP6o/yDpU1RlVNV5V42Nj\nYzslnCRpesMshklgfc/0OmD7TOskWQ08CPjxEDNJkuYwzGK4DNiY5KAkuwPHA1v61tkCnNjefx5w\ncZXnCkkrxZo1o07QNVOepZZz1IZ2umpV3Z3ktcCFNKernllVW5O8BZioqi3A+4Czk2yjeaVw/M7P\nsbMfcdflz1LLnaekDmao72OoqguAC/rmvann/v8Dnj/MDJKk+fGdz5KkDotBktRhMUiSOiwGSVKH\nxSBJ6rAYJEkdFoMkqSPL7Y3GSXYA31/g5muBm3ZinMWyHHObefEsx9xmXjxTuR9eVQNdbG7ZFcN9\nkWSiqsZHnWO+lmNuMy+e5ZjbzItnIbkdSpIkdVgMkqSOXa0Yzhh1gAVajrnNvHiWY24zL555596l\njjFIkua2q71ikCTNwWKQJHXsMsWQZHOSa5JsS3LKqPNMJ8mZSW5M8p2eefsm+cck322/PniUGfsl\nWZ/kC0muSrI1ye+285d67j2SfD3JFW3uP2znH5Tk0jb3ee2nDy4pSVYl+WaSz7TTyyHzdUn+Ocm3\nkky085b6PrJPkvOTXN3u34cv5cxJHtP+fKdutyX5vYVk3iWKIckq4HTgKGATcEKSTaNNNa0PAJv7\n5p0CXFRVG4GL2uml5G7gdVX1WOAw4DXtz3ap5/45cGRVPQ54PLA5yWHAnwLvbnPfApw0wowz+V3g\nqp7p5ZAZ4OlV9fiec+qX+j7yHuBzVfXrwONofuZLNnNVXdP+fB8PPBH4GfAJFpK5qlb8DTgcuLBn\n+lTg1FHnmiHrBuA7PdPXAPu39/cHrhl1xjnyfwp41nLKDewFfAP4LZp3iK6ebr9ZCjdgXfuf+0jg\nM0CWeuY213XA2r55S3YfAfYGvkd7gs5yyNyX89nAVxeaeZd4xQAcCFzfMz3ZzlsOHlpVPwRovz5k\nxHlmlGQDcAhwKcsgdzsk8y3gRuAfgX8Fbq2qu9tVluJ+8hfAG4BftNP7sfQzAxTw+SSXJzm5nbeU\n95FHADuA97fDdu9N8gCWduZexwPntPfnnXlXKYZMM8/zdHeiJA8EPgb8XlUti49cr6p7qnnZvQ44\nFHjsdKstbqqZJTkGuLGqLu+dPc2qSyZzj6dU1RNohnNfk+Spow40h9XAE4C/qapDgJ+yhIaNZtMe\nYzoW+OhCH2NXKYZJYH3P9Dpg+4iyzNcNSfYHaL/eOOI895JkN5pS+FBVfbydveRzT6mqW4Ev0hwj\n2SfJ6nbRUttPngIcm+Q64Fya4aS/YGlnBqCqtrdfb6QZ9z6Upb2PTAKTVXVpO30+TVEs5cxTjgK+\nUVU3tNPzzryrFMNlwMb27I3daV5mbRlxpkFtAU5s759IM4a/ZCQJ8D7gqqr6855FSz33WJJ92vt7\nAs+kObj4BeB57WpLKndVnVpV66pqA80+fHFVvYglnBkgyQOSrJm6TzP+/R2W8D5SVT8Crk/ymHbW\nM4ArWcKZe5zAr4aRYCGZR32QZBEPxhwN/AvNOPIbR51nhoznAD8E7qL5i+UkmjHki4Dvtl/3HXXO\nvsz/kWbo4tvAt9rb0csg98HAN9vc3wHe1M5/BPB1YBvNS/H7jzrrDPmPAD6zHDK3+a5ob1un/v8t\ng33k8cBEu498EnjwMsi8F3Az8KCeefPO7CUxJEkdu8pQkiRpQBaDJKnDYpAkdVgMkqQOi0GS1GEx\nSANKck971corknwjyZPb+RuSVJK39qy7NsldSf6qnT4tyetHlV2aD4tBGtwd1Vy98nE0F2J8W8+y\na4FjeqafT3POvrTsWAzSwuxNc4nrKXcAVyWZuqT0C4GPLHoqaSdYPfcqklp7tldj3YPm8sVH9i0/\nFzg+yY+Ae2iuWXTA4kaU7juLQRrcHdVcjZUkhwMfTPKbPcs/B7wVuAE4bwT5pJ3CoSRpAarqa8Ba\nYKxn3p3A5cDraK42Ky1LvmKQFiDJrwOraC5YtlfPoj8DvlRVNzcXnpWWH4tBGtzUMQZoPiDnxKq6\np7cAqmorno2kZc6rq0qSOjzGIEnqsBgkSR0WgySpw2KQJHVYDJKkDotBktRhMUiSOv4/a5qbjXbH\nX9sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1474f550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAF5BJREFUeJzt3XuYZHV95/H3hxkIIoMItAaY0UEd\njaNB0ZaA7KOIl4BBSAwqs+IVZddHY7Kr8dHNRpHERNHEuFlMQhRFoiCixpFg0IiX1Q1II6IZLskE\nEdoRGeTqZbn53T/O6WOlp6e7euzq6mrfr+fpZ+qc8zt1vr/pqvr0+Z1LpaqQJAlgp2EXIElaOgwF\nSVLHUJAkdQwFSVLHUJAkdQwFSVLHUNDISvLFJK/YzrLPJHnJgLb7qiTfT/LDJHsPYhs7KsnhSSZ7\npjclOXyIJWnErBx2AfrFkOQ64BVV9U+Lsb2qOmoQz5tkZ+DPgUOq6opBbGOe9RSwrqo2z7S8qh6z\nyCVpxLmnIM3Pg4FdgU3zXTGNkXjPJfEPxl9QI/EC1fKW5JVJNie5JcnGJPv1LHtykkuT3N7+++Tt\nPMe+Sb6Z5PXtdDe0lOSlSb6S5F1Jbk3y7SRH9ax7QJIvJ7kzyT8lOS3J382wjUcC17STtyW5aK4a\n2zreluSrwI+Bh83wvI9u293WDvccM239V/RMvzTJV9rHX25nX9EOZb1ghue+Lskz2sc7JXljkn9P\n8oMk5ybZq122NkklOTHJ9cBFSXZN8ndt29vavj14pv9/LR+GgoYqyRHAnwLPB/YFvgOc0y7bC/gH\n4H8Be9MM2/zD9HH8JGuBLwH/u6retZ1N/RrNB/o+wKnA+5OkXfYR4GvtNk4GXjTTE1TVvwJTwzF7\nVtURfdb4IuAkYFXbv97adwY+DXwWeBDwO8CHkzxqO/3orecp7cPHVdXuVfXROVZ5LfCbwFOB/YBb\ngdOmtXkq8Gjg14GXAA8A1rR9+6/AT+aqS6PNUNCwvRA4o6q+XlV3AW8CDm0/6H8D+LeqOquq7q2q\ns4Grgef0rL8e+CLwlqo6fZbtfKeq/raq7gPOpAmgByd5CPAk4M1VdXdVfQXYOI/6+6nxg1W1qV1+\nz7T1DwF2B97ebv8i4Hxgwzxq6Nd/Af6gqibb/+uTgeOmDRWdXFU/qqqfAPfQhMEjquq+qrqsqu4Y\nQF1aQgwFDdt+9Pz1XFU/BH4A7D99Wes77bIpLwS+C5w3x3Zu7NnGj9uHu7fbuKVnHsANO1r/dmqc\n7fn2A26oqp/Osv5CeSjwyXYo6DbgKuA+muMkU3prPQu4EDgnyZYkp7Z7NlrGDAUN2xaaDysAktyf\n5q/T705f1npIu2zKycDNwEeSrNiB7X8P2CvJbj3z1sxj/X5qnO1WxFuANdMOQPeu/yOgt7Zfnkdt\n090AHFVVe/b87FpVM9ZaVfdU1Vuraj3wZOBo4MU/x/Y1AgwFLaad24OXUz8racbzX5bk8Ul+CfgT\n4JKqug64AHhkkv+cZGV7IHU9zfDKlHuA5wH3B86a79k9VfUdYAI4OckuSQ7lPw79zKWfGmdzCc0H\n/xuS7NxeU/Ac2uMqwDeA5ybZLckjgBOnrf99Zjh4vR1/DbwtyUMBkowlOXZ7jZM8LcmvtmF7B83/\n9X19bksjylDQYrqA5kDl1M/JVfV54A+Bj9P81f5w4HiAqvoBzV+nr6MZUnoDcHRV3dz7pFV1N/Bc\nmgO1Z+zAaZ8vBA5tt/HHwEeBu/pZsd8aZ1n/buAY4CiaPZ73Ai+uqqvbJu8G7qb58D8T+PC0pzgZ\nOLMdEnr+HJt7D83xks8muRO4mOYA/Pb8Ms2w3B00Q01fArY5K0vLS/ySHek/SvJR4Oqqesuwa5EW\nm3sK+oWX5ElJHt6ex38kcCzw98OuSxoGr1qUmmGST9Ac4J4EXlVVlw+3JGk4HD6SJHUcPpIkdUZu\n+GifffaptWvXDrsMSRopl1122c1VNTZXu5ELhbVr1zIxMTHsMiRppCSZfuX9jBw+kiR1DAVJUsdQ\nkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUmdgF68lOYPmPvM3VdVjZ1gemvu7Pxv4MfDSqvr6wtex\n0M8oadUquGMHv615Pu/Jfrazxx5w5507tq62Ncg9hQ8CR86y/ChgXftzEvBXA6xF0gKa6UN4WNvZ\nXpvFqnG5GVgoVNWXgVtmaXIs8KFqXAzsmWTfQdUjSZrbMI8p7E/zReJTJtt520hyUpKJJBNbt25d\nlOIk6RfRMENhppHFGb/coapOr6rxqhofG5vzJn+SpB00zFCYBNb0TK8GtgypFkkSww2FjcCL0zgE\nuL2qvjfEeiT1adWqpbOd7bVZrBqXm0Gekno2cDiwT5JJ4C3AzgBV9dfABTSno26mOSX1ZYOow28b\nlZaWhX5PetrpwhpYKFTVhjmWF/DqQW1fkjR/XtEsSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEg\nSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoY\nCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoMNBSSHJnkmiSbk7xx\nhuUPSfKFJJcn+WaSZw+yHknS7AYWCklWAKcBRwHrgQ1J1k9r9j+Bc6vqIOB44L2DqkeSNLdB7ikc\nDGyuqmur6m7gHODYaW0K2KN9/ABgywDrkSTNYZChsD9wQ8/0ZDuv18nACUkmgQuA35npiZKclGQi\nycTWrVsHUaskicGGQmaYV9OmNwAfrKrVwLOBs5JsU1NVnV5V41U1PjY2NoBSJUkw2FCYBNb0TK9m\n2+GhE4FzAarqn4FdgX0GWJMkaRaDDIVLgXVJDkiyC82B5I3T2lwPPB0gyaNpQsHxIUkakoGFQlXd\nC7wGuBC4iuYso01JTklyTNvsdcArk1wBnA28tKqmDzFJkhbJykE+eVVdQHMAuXfem3seXwkcNsga\nJEn984pmSVLHUJAkdQwFSVLHUJAkdQwFSVLHUJAkdQwFSVLHUJAkdQwFSVLHUJAkdQwFSVLHUJAk\ndQwFSVLHUJAkdQwFSVLHUJAkdQwFSVLHUJAkdQwFSVLHUJAkdQwFSVLHUJAkdQwFSVLHUJAkdQwF\nSVLHUJAkdQwFSVLHUJAkdQwFSVLHUJAkdQYaCkmOTHJNks1J3ridNs9PcmWSTUk+Msh6JEmzWzmo\nJ06yAjgNeCYwCVyaZGNVXdnTZh3wJuCwqro1yYMGVY8kaW6D3FM4GNhcVddW1d3AOcCx09q8Ejit\nqm4FqKqbBliPJGkOfYVCkt2S/GGSv22n1yU5eo7V9gdu6JmebOf1eiTwyCRfTXJxkiO3s/2Tkkwk\nmdi6dWs/JUuSdkC/ewofAO4CDm2nJ4E/nmOdzDCvpk2vBNYBhwMbgPcl2XOblapOr6rxqhofGxvr\ns2RJ0nz1GwoPr6pTgXsAquonzPyh32sSWNMzvRrYMkObT1XVPVX1beAampCQJA1Bv6Fwd5L70f6l\nn+ThNHsOs7kUWJfkgCS7AMcDG6e1+Xvgae1z7kMznHRtnzVJkhZYv2cfvQX4R2BNkg8DhwEvnW2F\nqro3yWuAC4EVwBlVtSnJKcBEVW1slz0ryZXAfcDvV9UPdqwrkqSfV6qmD/Nvp2GyN3AIzbDRxVV1\n8yAL257x8fGamJgYxqYlaWQluayqxudqN59TUven+Yt/F+ApSZ67o8VJkpamvoaPkpwBHAhsAn7a\nzi7gEwOqS5I0BP0eUzikqtYPtBJJ0tD1O3z0z0kMBUla5vrdUziTJhhupDkVNUBV1YEDq0yStOj6\nDYUzgBcB3+JnxxQkSctMv6FwfXtdgSRpGes3FK5uv+vg0/RcyVxVnn0kSctIv6FwP5oweFbPPE9J\nlaRlpq9QqKqXDboQSdLw9ft9CquTfDLJTUm+n+TjSVYPujhJ0uKaz/cpbAT2o7ndxafbeZKkZaTf\nUBirqg9U1b3tzwcBv+1GkpaZfkPh5iQnJFnR/pwAeItrSVpm+g2FlwPPB24Evgcc186TJC0j/Z59\ndD1wzIBrkSQNWb9nH52ZZM+e6Qe2t9OWJC0j/Q4fHVhVt01NVNWtwEGDKUmSNCz9hsJOSR44NZFk\nL/q/GlqSNCL6/WD/M+D/JjmP5vYWzwf+ZGBVSZKGot8DzR9KMgEcQfNdCs+tqisHWpkkadH1+x3N\nZ1XVi4ArZ5gnSVom+j2m8JjeiSQrgCcufDmSpGGaNRSSvCnJncCBSe5Icmc7fRPwqUWpUJK0aGYN\nhar606paBbyzqvaoqlXtz95V9aZFqlGStEj6PfvoM0meMn1mVX15geuRJA1Rv6Hw+z2PdwUOBi6j\nORtJkrRM9HtK6nN6p5OsAU4dSEWSpKHp9+yj6SaBxy5kIZKk4ev3OoW/pLmSGZogOQi4oo/1jgTe\nA6wA3ldVb99Ou+OAjwFPqqqJfmqSJC28fo8pXEnzwV7A7cDZVfXV2VZor2U4DXgmzZ7FpUk2Tr8S\nOskq4LXAJfOsXZK0wGYNhSQrae5x9HLgeppbXKwBzkjytaq6Z5bVDwY2V9W17XOdAxxLz1XRrT+i\nOT7x+h3qgSRpwcx1TOGdwF7AAVX1hKo6CHgYsCfwrjnW3R+4oWd6sp3XSXIQsKaqzp/tiZKclGQi\nycTWrVvn2KwkaUfNFQpHA6+sqjunZlTVHcCrgGfPsW5mmFfdwmQn4N3A6+YqsqpOr6rxqhofGxub\nq7kkaQfNFQpVVTXDzPvo+YDfjkmaoaYpq4EtPdOraM5g+mKS64BDgI1JxucqWpI0GHOFwpVJXjx9\nZpITgKvnWPdSYF2SA5LsAhwPbJxaWFW3V9U+VbW2qtYCFwPHePaRJA3PXGcfvRr4RJKX01zBXMCT\ngPsBvzXbilV1b5LXABfSnLl0RlVtSnIKMFFVG2dbX5K0+DLD6NC2jZIjaG6fHWBTVX1+0IVtz/j4\neE1MuDMhSfOR5LKqmnN4vt/bXFwEXPRzVyVJWtJ29DYXkqRlyFCQJHUMBUlSx1CQJHUMBUlSx1CQ\nJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUM\nBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSZ6ChkOTIJNck\n2ZzkjTMs/+9JrkzyzSSfT/LQQdYjSZrdwEIhyQrgNOAoYD2wIcn6ac0uB8ar6kDgPODUQdUjSZrb\nIPcUDgY2V9W1VXU3cA5wbG+DqvpCVf24nbwYWD3AeiRJcxhkKOwP3NAzPdnO254Tgc/MtCDJSUkm\nkkxs3bp1AUuUJPUaZChkhnk1Y8PkBGAceOdMy6vq9Koar6rxsbGxBSxRktRr5QCfexJY0zO9Gtgy\nvVGSZwB/ADy1qu4aYD2SpDkMck/hUmBdkgOS7AIcD2zsbZDkIOBvgGOq6qYB1iJJ6sPAQqGq7gVe\nA1wIXAWcW1WbkpyS5Ji22TuB3YGPJflGko3beTpJ0iIY5PARVXUBcMG0eW/uefyMQW5fkjQ/XtEs\nSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoY\nCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKk\njqEgSeoYCpKkjqEgSeqsHOSTJzkSeA+wAnhfVb192vJfAj4EPBH4AfCCqrpuYWtYyGeTtFRVzb58\njz3gzju3nb9qFdxxx2BqWgiLXffA9hSSrABOA44C1gMbkqyf1uxE4NaqegTwbuAdg6pH0i+2mT5Y\nZ5u/VCx23YMcPjoY2FxV11bV3cA5wLHT2hwLnNk+Pg94euLf9pI0LIMMhf2BG3qmJ9t5M7apqnuB\n24G9pz9RkpOSTCSZ2Lp164DKlSQNMhRm+ot/+qhfP22oqtOraryqxsfGxhakOEnStgYZCpPAmp7p\n1cCW7bVJshJ4AHDLAGuSJM1ikKFwKbAuyQFJdgGOBzZOa7MReEn7+Djgoqq5ziGQpPlbtWp+85eK\nxa57YKekVtW9SV4DXEhzSuoZVbUpySnARFVtBN4PnJVkM80ewvELX8dCP6OkUbSUTzudzWLXPdDr\nFKrqAuCCafPe3PP4/wHPG2QNkqT+eUWzJKljKEiSOoaCJKljKEiSOoaCJKljKEiSOoaCJKmTUbuA\nOMlW4Dt9NN0HuHnA5SwW+7L0LJd+gH1Zqha6Lw+tqjlvHjdyodCvJBNVNT7sOhaCfVl6lks/wL4s\nVcPqi8NHkqSOoSBJ6iznUDh92AUsIPuy9CyXfoB9WaqG0pdle0xBkjR/y3lPQZI0T4aCJKmzLEIh\nyZokX0hyVZJNSX63nb9Xks8l+bf23wcOu9bZJNk1ydeSXNH2463t/AOSXNL246PtN9mNhCQrklye\n5Px2eiT7kuS6JN9K8o0kE+28kXp9TUmyZ5LzklzdvmcOHbW+JHlU+7uY+rkjye+NWj+mJPlv7Xv+\nX5Kc3X4WDOW9sixCAbgXeF1VPRo4BHh1kvXAG4HPV9U64PPt9FJ2F3BEVT0OeDxwZJJDgHcA7277\ncStw4hBrnK/fBa7qmR7lvjytqh7fc+74qL2+prwH+Meq+hXgcTS/n5HqS1Vd0/4uHg88Efgx8ElG\nrB8ASfYHXguMV9Vjab6p8niG9V6pqmX3A3wKeCZwDbBvO29f4Jph1zaPPuwGfB34NZqrGle28w8F\nLhx2fX32YTXNG/MI4HwgI9yX64B9ps0budcXsAfwbdqTTEa5Lz21Pwv46qj2A9gfuAHYi+bbMM8H\nfn1Y75XlsqfQSbIWOAi4BHhwVX0PoP33QcOrrD/tcMs3gJuAzwH/DtxWVfe2TSZpXkSj4C+ANwA/\nbaf3ZnT7UsBnk1yW5KR23si9voCHAVuBD7TDeu9Lcn9Gsy9TjgfObh+PXD+q6rvAu4Drge8BtwOX\nMaT3yrIKhSS7Ax8Hfq+qRvJruqvqvmp2iVcDBwOPnqnZ4lY1f0mOBm6qqst6Z8/QdMn3pXVYVT0B\nOIpmePIpwy5oB60EngD8VVUdBPyIERhi2Z52nP0Y4GPDrmVHtcc9jgUOAPYD7k/zOptuUd4ryyYU\nkuxMEwgfrqpPtLO/n2Tfdvm+NH99j4Squg34Is0xkj2TrGwXrQa2DKuueTgMOCbJdcA5NENIf8Fo\n9oWq2tL+exPN2PXBjObraxKYrKpL2unzaEJiFPsCzYfn16vq++30KPbjGcC3q2prVd0DfAJ4MkN6\nryyLUEgS4P3AVVX15z2LNgIvaR+/hOZYw5KVZCzJnu3j+9G8WK4CvgAc1zZb8v0AqKo3VdXqqlpL\ns3t/UVW9kBHsS5L7J1k19ZhmDPtfGLHXF0BV3QjckORR7aynA1cygn1pbeBnQ0cwmv24HjgkyW7t\nZ9nU72Qo75VlcUVzkv8E/B/gW/xs/Pp/0BxXOBd4CM1//POq6pahFNmHJAcCZ9KcfbATcG5VnZLk\nYTR/be8FXA6cUFV3Da/S+UlyOPD6qjp6FPvS1vzJdnIl8JGqeluSvRmh19eUJI8H3gfsAlwLvIz2\n9cYI9SXJbjQHaB9WVbe380b1d/JW4AU0Z1JeDryC5hjCor9XlkUoSJIWxrIYPpIkLQxDQZLUMRQk\nSR1DQZLUMRQkSR1DQZqHJL+VpJL8yrBrkQbBUJDmZwPwFZoL8qRlx1CQ+tTeW+swmlsYH9/O2ynJ\ne9t74Z+f5IIkx7XLnpjkS+1N9C6cuv2CtJQZClL/fpPmewj+FbglyROA5wJrgV+luQr1UOjuxfWX\nwHFV9UTgDOBtwyhamo+VczeR1NpAc1M/aG4/sAHYGfhYVf0UuDHJF9rljwIeC3yuuZ0NK2huiywt\naYaC1If2njpHAI9NUjQf8sXP7om0zSrApqo6dJFKlBaEw0dSf44DPlRVD62qtVW1huYbzG4Gfrs9\ntvBg4PC2/TXAWJJuOCnJY4ZRuDQfhoLUnw1su1fwcZovRZmkuZX239Dcmff2qrqbJkjekeQK4Bs0\n98iXljTvkir9nJLsXlU/bIeYvkbzLW03DrsuaUd4TEH6+Z3ffjnSLsAfGQgaZe4pSJI6HlOQJHUM\nBUlSx1CQJHUMBUlSx1CQJHX+P36rF/4ykGk4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x147f1550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 离群点检测（outliers）\n",
    "plt.scatter(train.SkinThickness, train.Outcome, c = \"blue\", marker = \"s\")\n",
    "plt.title(\"Looking for outliers\")\n",
    "plt.xlabel(\"SkinThickness\")\n",
    "plt.ylabel(\"Outcome\")\n",
    "plt.show()\n",
    "\n",
    "#剔除离群点\n",
    "train = train[train.SkinThickness < 70]\n",
    "temp = train.reindex()\n",
    "\n",
    "# 离群点检测（outliers）\n",
    "plt.scatter(train.Insulin, train.Outcome, c = \"blue\", marker = \"s\")\n",
    "plt.title(\"Looking for outliers\")\n",
    "plt.xlabel(\"Insulin\")\n",
    "plt.ylabel(\"Outcome\")\n",
    "plt.show()\n",
    "#剔除离群点\n",
    "train = train[train.Insulin < 680]\n",
    "temp = train.reindex()\n",
    "\n",
    "# 离群点检测（outliers）\n",
    "plt.scatter(train.BMI, train.Outcome, c = \"blue\", marker = \"s\")\n",
    "plt.title(\"Looking for outliers\")\n",
    "plt.xlabel(\"BMI\")\n",
    "plt.ylabel(\"Outcome\")\n",
    "plt.show()\n",
    "#剔除离群点\n",
    "train = train[train.BMI < 65]\n",
    "temp = train.reindex()\n",
    "\n",
    "# 离群点检测（outliers）\n",
    "plt.scatter(train.Age, train.Outcome, c = \"blue\", marker = \"s\")\n",
    "plt.title(\"Looking for outliers\")\n",
    "plt.xlabel(\"Age\")\n",
    "plt.ylabel(\"Outcome\")\n",
    "plt.show()\n",
    "#剔除离群点\n",
    "train = train[train.Age < 80]\n",
    "temp = train.reindex()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>762.000000</td>\n",
       "      <td>762.000000</td>\n",
       "      <td>762.000000</td>\n",
       "      <td>762.000000</td>\n",
       "      <td>762.000000</td>\n",
       "      <td>762.000000</td>\n",
       "      <td>762.000000</td>\n",
       "      <td>762.000000</td>\n",
       "      <td>762.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.854331</td>\n",
       "      <td>120.519685</td>\n",
       "      <td>69.027559</td>\n",
       "      <td>20.339895</td>\n",
       "      <td>77.199475</td>\n",
       "      <td>31.920604</td>\n",
       "      <td>0.469676</td>\n",
       "      <td>33.132546</td>\n",
       "      <td>0.347769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.369655</td>\n",
       "      <td>31.717217</td>\n",
       "      <td>19.356685</td>\n",
       "      <td>15.704606</td>\n",
       "      <td>107.435570</td>\n",
       "      <td>7.771173</td>\n",
       "      <td>0.325657</td>\n",
       "      <td>11.585006</td>\n",
       "      <td>0.476575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>27.300000</td>\n",
       "      <td>0.241500</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>0.370000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>140.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>126.000000</td>\n",
       "      <td>36.500000</td>\n",
       "      <td>0.626750</td>\n",
       "      <td>40.750000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>199.000000</td>\n",
       "      <td>122.000000</td>\n",
       "      <td>63.000000</td>\n",
       "      <td>600.000000</td>\n",
       "      <td>59.400000</td>\n",
       "      <td>2.420000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Pregnancies     Glucose  BloodPressure  SkinThickness     Insulin  \\\n",
       "count   762.000000  762.000000     762.000000     762.000000  762.000000   \n",
       "mean      3.854331  120.519685      69.027559      20.339895   77.199475   \n",
       "std       3.369655   31.717217      19.356685      15.704606  107.435570   \n",
       "min       0.000000    0.000000       0.000000       0.000000    0.000000   \n",
       "25%       1.000000   99.000000      62.000000       0.000000    0.000000   \n",
       "50%       3.000000  117.000000      72.000000      23.000000   24.000000   \n",
       "75%       6.000000  140.000000      80.000000      32.000000  126.000000   \n",
       "max      17.000000  199.000000     122.000000      63.000000  600.000000   \n",
       "\n",
       "              BMI  DiabetesPedigreeFunction         Age     Outcome  \n",
       "count  762.000000                762.000000  762.000000  762.000000  \n",
       "mean    31.920604                  0.469676   33.132546    0.347769  \n",
       "std      7.771173                  0.325657   11.585006    0.476575  \n",
       "min      0.000000                  0.078000   21.000000    0.000000  \n",
       "25%     27.300000                  0.241500   24.000000    0.000000  \n",
       "50%     32.000000                  0.370000   29.000000    0.000000  \n",
       "75%     36.500000                  0.626750   40.750000    1.000000  \n",
       "max     59.400000                  2.420000   72.000000    1.000000  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 查看特征列中0元素的数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pregnancies = 109\n",
      "Glucose = 5\n",
      "BloodPressure = 35\n",
      "SkinThickness = 227\n",
      "Insulin = 373\n",
      "BMI = 11\n",
      "DiabetesPedigreeFunction = 0\n",
      "Age = 0\n",
      "Outcome = 497\n"
     ]
    }
   ],
   "source": [
    "for a in train.columns:\n",
    "    print('%s = %d'% (a ,len(train[train[a]==0])))\n",
    "    #m=train[a].mean\n",
    "    #train[a].replace({0:m},inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 中位数填充0位置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a=117\n",
    "train.Glucose.replace({0:a},inplace = True)\n",
    "a=72\n",
    "train.BloodPressure.replace({0 : a},inplace = True)\n",
    "a=23\n",
    "train.SkinThickness.replace({0 : a},inplace = True)\n",
    "a=24\n",
    "train.Insulin.replace({0 : a},inplace = True)\n",
    "a=32\n",
    "train.BMI.replace({0 : a},inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pregnancies = 109\n",
      "Glucose = 0\n",
      "BloodPressure = 0\n",
      "SkinThickness = 0\n",
      "Insulin = 0\n",
      "BMI = 0\n",
      "DiabetesPedigreeFunction = 0\n",
      "Age = 0\n",
      "Outcome = 497\n"
     ]
    }
   ],
   "source": [
    "for a in train.columns:\n",
    "    print('%s = %d'% (a ,len(train[train[a]==0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 762 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "Pregnancies                 762 non-null int64\n",
      "Glucose                     762 non-null int64\n",
      "BloodPressure               762 non-null int64\n",
      "SkinThickness               762 non-null int64\n",
      "Insulin                     762 non-null int64\n",
      "BMI                         762 non-null float64\n",
      "DiabetesPedigreeFunction    762 non-null float64\n",
      "Age                         762 non-null int64\n",
      "Outcome                     762 non-null int64\n",
      "dtypes: float64(2), int64(7)\n",
      "memory usage: 59.5 KB\n"
     ]
    }
   ],
   "source": [
    "train.info()\n",
    "#type(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 划分训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 将类别字符串变成数字\n",
    "# drop ids and get labels\n",
    "Y = train['Outcome']   #形式为Class_x\n",
    "\n",
    "X = train.drop([\"Outcome\"], axis=1)\n",
    "X = np.array(X)\n",
    "\n",
    "#如果计算资源有限，也可只取少量样本，如取前1000个样本\n",
    "#（分类中其实还需要确保取出来的这部分样本各类样本的比例和总体一致）\n",
    "\n",
    "#或者考虑用train_test_split而不是交叉验证来验证模型性能"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据集划分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def data_set_split(X,Y):\n",
    "    from sklearn.cross_validation import train_test_split\n",
    "    #random_state=42 保证每次分组一致，保证模型稳定\n",
    "    X_train, X_test, y_train,y_test = train_test_split(X, Y, test_size=0.2) \n",
    "    return X_train, X_test, y_train, y_test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test=data_set_split(X,Y)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征进行标准化处理\n",
    "X_train = ss_X.fit_transform(X_train)\n",
    "X_test = ss_X.transform(X_test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将0.2的数据作为测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(609, 8) (153, 8)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEn5JREFUeJzt3XuQJWV9xvHvAwuieFmQkUJgs5BC\nS8oL6moBJqhAKt4CSMBoUbiFVFZLRUyMAY0RFeMl4jVGzZagS2JAJCCoKCoCRg2ry0XlIgUiwgqy\nA4JcTEDglz9Oj4xrz2wzZ8+Fme+naup0v6fPeX9btTvPdr/9vp2qQpKk9W0y6gIkSePJgJAktTIg\nJEmtDAhJUisDQpLUyoCQJLUyICRJrQwISVIrA0KS1GrRqAvoxzbbbFNLly4ddRmS9JBy4YUX3lxV\nExs67iEdEEuXLmXNmjWjLkOSHlKS/LzLcV5ikiS1MiAkSa0MCElSKwNCktTKgJAktTIgJEmtDAhJ\nUquBBUSSE5KsS3LptLatk3wjyVXN61ZNe5J8LMnVSX6U5BmDqkuS1M0gzyA+C7xgvbajgXOqahfg\nnGYf4IXALs3PCuCTA6xLktTBwGZSV9W3kyxdr3l/4HnN9irgPOCopv3EqirggiSLk2xXVTcOqr4p\nz3zziYPuQg9BF37glaMuQRq5YY9BbDv1S795fVzTvj1w/bTj1jZtkqQRGZdB6rS0VeuByYoka5Ks\nmZycHHBZkrRwDTsgbkqyHUDzuq5pXwvsOO24HYAb2r6gqlZW1bKqWjYxscHFCCVJczTsgDgTWN5s\nLwfOmNb+yuZupt2BXw9j/EGSNLOBDVInOYnegPQ2SdYCxwDvA05JcjhwHXBwc/hZwIuAq4HfAIcN\nqi5JUjeDvIvpFTO8tU/LsQW8blC1SJIevHEZpJYkjRkDQpLUyoCQJLUyICRJrQwISVIrA0KS1MqA\nkCS1MiAkSa0MCElSKwNCktTKgJAktTIgJEmtDAhJUisDQpLUyoCQJLUyICRJrQwISVIrA0KS1MqA\nkCS1MiAkSa0MCElSKwNCktTKgJAktTIgJEmtDAhJUisDQpLUyoCQJLUyICRJrQwISVIrA0KS1MqA\nkCS1MiAkSa1GEhBJ/ibJZUkuTXJSki2S7JRkdZKrknw+yeajqE2S1DP0gEiyPfAGYFlVPRnYFHg5\n8H7gw1W1C3ArcPiwa5MkPWCDAZFkyySbNNtPSLJfks367HcR8PAki4BHADcCewOnNu+vAg7osw9J\nUh+6nEF8G9ii+Z//OcBhwGfn2mFV/QI4DriOXjD8GrgQuK2q7m0OWwtsP9c+JEn96xIQqarfAAcC\n/1JVLwV2nWuHSbYC9gd2Ah4PbAm8sOXQmuHzK5KsSbJmcnJyrmVIkjagU0Ak2QM4BPhK07aojz73\nBX5WVZNV9VvgNGBPYHFzyQlgB+CGtg9X1cqqWlZVyyYmJvooQ5I0my4B8UbgLcDpVXVZkp2Bc/vo\n8zpg9ySPSBJgH+Dy5jsPao5ZDpzRRx+SpD5t8Eygqs4Hzk+yZbN/Db27kOakqlYnORW4CLgXuBhY\nSe/s5OQk727ajp9rH5Kk/m0wIJrLS8cDjwSWJHka8Oqqeu1cO62qY4Bj1mu+Bnj2XL9TkrRxdbnE\n9BHgz4FbAKrqh8BegyxKkjR6nSbKVdX16zXdN4BaJEljpMvdSNcn2ROoZvmLNwBXDLYsSdKodTmD\neA3wOnoT19YCuzX7kqR5rMtdTDfTmwMhSVpAuqzFtCrJ4mn7WyU5YbBlSZJGrcslpqdW1W1TO1V1\nK/D0wZUkSRoHXQJik2b9JACSbE1/S21Ikh4Cuvyi/yDwvWb2M8DBwD8NriRJ0jjoMkh9YpILgecD\nAQ6sqssHXpkkaaS6Xir6Cb2nvC0CSLKkqq4bWFWSpJHrshbTEfTWTbqJ3gzq0HtWw1MHW5okaZS6\nnEEcCTyxqm4ZdDGSpPHR5S6m6+k9FlSStIB0OYO4BjgvyVeAu6caq+pDA6tKkjRyXQLiuuZn8+ZH\nkrQAdLnN9Z0ASbasqrsGX5IkaRx0WYtpjySX0yzxneRpST4x8MokSSPlE+UkSa18opwkqZVPlJMk\ntfKJcpKkVrOeQSTZFDi0qnyinCQtMLOeQVTVfcD+Q6pFkjRGuoxBfDfJx4HPA7+bB1FVFw2sKknS\nyHUJiD2b13dNaytg741fjiRpXGxoDGIT4JNVdcqQ6pEkjYkNjUHcD7x+SLVIksZIl9tcv5Hk75Ls\nmGTrqZ+BVyZJGqkuYxCval6nz30oYOeNX44kaVx0Wc11p2EUIkkaL12eSf3KtvaqOnHjlyNJGhdd\nLjE9a9r2FsA+wEXAnAMiyWLg08CT6V2uehVwJb25FkuBa4GXVdWtc+1DktSfLpeYjpi+n+QxwL/3\n2e9Hga9V1UHNAoCPAN4KnFNV70tyNHA0cFSf/UiS5qjTct/r+Q2wy1w7TPJoes+TOB6gqu6pqtvo\nLemxqjlsFXDAXPuQJPWvyxjEl+hdBoJeoOwK9DNxbmdgEvhMkqcBFwJHAttW1Y0AVXVjksf10Yck\nqU9dxiCOm7Z9L/DzqlrbZ5/PAI6oqtVJPkrvclInSVYAKwCWLFnSRxmSpNl0ucR0HbC6qs6vqu8C\ntyRZ2kefa4G1VbW62T+VXmDclGQ7gOZ1XduHq2plVS2rqmUTExN9lCFJmk2XgPgCcP+0/fuatjmp\nql/Se0rdE5umfYDLgTOB5U3bcuCMufYhSepfl0tMi6rqnqmdqrqnufOoH0cAn2u+5xrgMHphdUqS\nw+mdtRzcZx+SpD50CYjJJPtV1ZkASfYHbu6n06q6BFjW8tY+/XyvJGnj6RIQr6H3v/2PN/trgdbZ\n1ZKk+aPLRLmfArsneSSQqrpj8GVJkkZtg4PUSd6TZHFV3VlVdyTZKsm7h1GcJGl0utzF9MJmpjMA\nzfpILxpcSZKkcdAlIDZN8rCpnSQPBx42y/GSpHmgyyD1fwDnJPkMD6y8umr2j0iSHuq6DFL/c5If\nAfs2TcdW1dmDLUuSNGpdziAALgY2o3cGcfHgypEkjYsudzG9DPg+cBDwMmB1koMGXZgkabS6nEH8\nA/CsqloHkGQC+Ca9RfYkSfNUl7uYNpkKh8YtHT8nSXoI63IG8bUkZwMnNft/BZw1uJIkAVz3rqeM\nugSNoSVv//HQ+upyF9ObkxwI/AkQYGVVnT7wyiRJI9XpLqaqOg04bcC1SJLGiGMJkqRWBoQkqdWM\nAZHknOb1/cMrR5I0LmYbg9guyXOB/ZKcTG+A+neq6qKBViZJGqnZAuLtwNHADsCH1nuvgL0HVZQk\nafRmDIiqOhU4Nck/VtWxQ6xJkjQGusyDODbJfsBeTdN5VfXlwZYlSRq1Lov1vRc4Eri8+TmyaZMk\nzWNdJsq9GNitqu4HSLKK3pLfbxlkYZKk0eo6D2LxtO3HDKIQSdJ46XIG8V7g4iTn0rvVdS88e5Ck\nea/LIPVJSc4DnkUvII6qql8OujBJ0mh1XazvRuDMAdciSRojrsUkSWplQEiSWs0aEEk2SXLpsIqR\nJI2PWQOimfvwwyRLhlSPJGlMdBmk3g64LMn3gbumGqtqv4FVJUkauS4B8c5BdJxkU2AN8IuqekmS\nnYCTga2Bi4BDq+qeQfQtSdqwDQ5SV9X5wLXAZs32D+j9Au/XkcAV0/bfD3y4qnYBbgUO3wh9SJLm\nqMtifX8NnAr8W9O0PfDFfjpNsgO9NZ4+3eyH3vMlTm0OWQUc0E8fkqT+dLnN9XXAc4DbAarqKuBx\nffb7EeDvgfub/ccCt1XVvc3+WnpBJEkakS4Bcff0sYAki+g9UW5OkrwEWFdVF05vbjm0tY8kK5Ks\nSbJmcnJyrmVIkjagS0Ccn+StwMOT/BnwBeBLffT5HHrPub6W3qD03vTOKBY34QO9x5ze0PbhqlpZ\nVcuqatnExEQfZUiSZtMlII4GJoEfA68GzgLeNtcOq+otVbVDVS0FXg58q6oOAc4FDmoOWw6cMdc+\nJEn967Ka6/3NQ4JW07vsc2VVzfkS0yyOAk5O8m56DyQ6fgB9SJI62mBAJHkx8Cngp/TGCnZK8uqq\n+mq/nVfVecB5zfY1wLP7/U5J0sbRZaLcB4HnV9XVAEn+GPgK0HdASJLGV5cxiHVT4dC4Blg3oHok\nSWNixjOIJAc2m5clOQs4hd4YxMH0ZlNLkuax2S4x/cW07ZuA5zbbk8BWA6tIkjQWZgyIqjpsmIVI\nksZLl7uYdgKOAJZOP97lviVpfutyF9MX6c1J+BIPrJ0kSZrnugTE/1XVxwZeiSRprHQJiI8mOQb4\nOnD3VGNVbYxnQkiSxlSXgHgKcCi9RfWmLjFVsy9Jmqe6BMRLgZ19/KckLSxdZlL/EFg86EIkSeOl\nyxnEtsBPkvyA3x+D8DZXSZrHugTEMQOvQpI0dro8D+L8YRQiSRovXWZS38EDz4feHNgMuKuqHj3I\nwiRJo9XlDOJR0/eTHIAP9pGkea/LXUy/p6q+iHMgJGne63KJ6cBpu5sAy3jgkpMkaZ7qchfT9OdC\n3AtcC+w/kGokSWOjyxiEz4WQpAVotkeOvn2Wz1VVHTuAeiRJY2K2M4i7Wtq2BA4HHgsYEJI0j832\nyNEPTm0neRRwJHAYcDLwwZk+J0maH2Ydg0iyNfC3wCHAKuAZVXXrMAqTJI3WbGMQHwAOBFYCT6mq\nO4dWlSRp5GabKPcm4PHA24Abktze/NyR5PbhlCdJGpXZxiAe9CxrSdL8YQhIkloZEJKkVgaEJKmV\nASFJajX0gEiyY5Jzk1yR5LIkRzbtWyf5RpKrmtethl2bJOkBoziDuBd4U1U9CdgdeF2SXYGjgXOq\nahfgnGZfkjQiQw+Iqrqxqi5qtu8ArgC2p7eE+KrmsFXAAcOuTZL0gJGOQSRZCjwdWA1sW1U3Qi9E\ngMeNrjJJ0sgCIskjgf8C3lhVnWdmJ1mRZE2SNZOTk4MrUJIWuJEERJLN6IXD56rqtKb5piTbNe9v\nB6xr+2xVrayqZVW1bGJiYjgFS9ICNIq7mAIcD1xRVR+a9taZwPJmezlwxrBrkyQ9oMszqTe25wCH\nAj9OcknT9lbgfcApSQ4HrgMOHkFtkqTG0AOiqr4DZIa39xlmLZKkmTmTWpLUyoCQJLUyICRJrQwI\nSVIrA0KS1MqAkCS1MiAkSa0MCElSKwNCktTKgJAktTIgJEmtDAhJUisDQpLUyoCQJLUyICRJrQwI\nSVIrA0KS1MqAkCS1MiAkSa0MCElSKwNCktTKgJAktTIgJEmtDAhJUisDQpLUyoCQJLUyICRJrQwI\nSVIrA0KS1MqAkCS1MiAkSa0MCElSq7EKiCQvSHJlkquTHD3qeiRpIRubgEiyKfCvwAuBXYFXJNl1\ntFVJ0sI1NgEBPBu4uqquqap7gJOB/UdckyQtWOMUENsD10/bX9u0SZJGYNGoC5gmLW31BwclK4AV\nze6dSa4caFULyzbAzaMuYhzkuOWjLkG/z7+bU45p+1X5oP1Rl4PGKSDWAjtO298BuGH9g6pqJbBy\nWEUtJEnWVNWyUdchrc+/m6MxTpeYfgDskmSnJJsDLwfOHHFNkrRgjc0ZRFXdm+T1wNnApsAJVXXZ\niMuSpAVrbAICoKrOAs4adR0LmJfuNK78uzkCqfqDcWBJksZqDEKSNEYMCLnEicZWkhOSrEty6ahr\nWYgMiAXOJU405j4LvGDURSxUBoRc4kRjq6q+Dfxq1HUsVAaEXOJEUisDQp2WOJG08BgQ6rTEiaSF\nx4CQS5xIamVALHBVdS8wtcTJFcApLnGicZHkJOB/gCcmWZvk8FHXtJA4k1qS1MozCElSKwNCktTK\ngJAktTIgJEmtDAhJUisDQppBksVJXjuEfp6XZM9B9yM9WAaENLPFQOeASM9c/k09DzAgNHacByHN\nIMnUyrZXAucCTwW2AjYD3lZVZyRZCny1eX8P4ABgX+AoekuWXAXcXVWvTzIBfApY0nTxRuAXwAXA\nfcAkcERV/fcw/nzShhgQ0gyaX/5frqonJ1kEPKKqbk+yDb1f6rsAfwRcA+xZVRckeTzwPeAZwB3A\nt4AfNgHxn8Anquo7SZYAZ1fVk5K8A7izqo4b9p9Rms2iURcgPUQEeE+SvYD76S2Jvm3z3s+r6oJm\n+9nA+VX1K4AkXwCe0Ly3L7Br8rsFdB+d5FHDKF6aCwNC6uYQYAJ4ZlX9Nsm1wBbNe3dNO65t+fQp\nmwB7VNX/Tm+cFhjSWHGQWprZHcDU//AfA6xrwuH59C4ttfk+8NwkWzWXpf5y2ntfp7cwIgBJdmvp\nRxobBoQ0g6q6BfhukkuB3YBlSdbQO5v4yQyf+QXwHmA18E3gcuDXzdtvaL7jR0kuB17TtH8JeGmS\nS5L86cD+QNKD5CC1tJEleWRV3dmcQZwOnFBVp4+6LunB8gxC2vjekeQS4FLgZ8AXR1yPNCeeQUiS\nWnkGIUlqZUBIkloZEJKkVgaEJKmVASFJamVASJJa/T89IqXPVxXJfwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14175470>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Target 分布，看看各类样本分布是否均衡\n",
    "sns.countplot(y_test);\n",
    "plt.xlabel('target');\n",
    "plt.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### default Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr= LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.54512894 0.46791974 0.47251922 0.4874715  0.42090136]\n",
      "cv logloss is: 0.4787881512559677\n"
     ]
    }
   ],
   "source": [
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化的 Logistic Regression及参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） \n",
    "目标函数为：J = sum(logloss(f(xi), yi)) + C* penalty \n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同：\n",
    "设置候选参数集合\n",
    "调用GridSearchCV\n",
    "调用fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.00079999, 0.00120006, 0.00060005, 0.00060005, 0.00039997,\n",
       "        0.        , 0.        , 0.00311999, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.00020003]),\n",
       " 'mean_score_time': array([0.00100012, 0.00080009, 0.00100002, 0.0006    , 0.00040002,\n",
       "        0.        , 0.        , 0.        , 0.00311999, 0.        ,\n",
       "        0.        , 0.        , 0.00311999, 0.00312004]),\n",
       " 'mean_test_score': array([-0.69314718, -0.64140232, -0.67351257, -0.52911181, -0.48527473,\n",
       "        -0.48007377, -0.47919976, -0.47897788, -0.47947627, -0.47946991,\n",
       "        -0.47952869, -0.47953059, -0.4795328 , -0.47953662]),\n",
       " 'mean_train_score': array([-0.69314718, -0.64014666, -0.6728074 , -0.52435224, -0.47787944,\n",
       "        -0.46965385, -0.46537698, -0.46523335, -0.46515557, -0.4651541 ,\n",
       "        -0.46515326, -0.46515324, -0.46515323, -0.46515323]),\n",
       " 'param_C': masked_array(data=[0.001, 0.001, 0.01, 0.01, 0.1, 0.1, 1, 1, 10, 10, 100,\n",
       "                    100, 1000, 1000],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_penalty': masked_array(data=['l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1',\n",
       "                    'l2', 'l1', 'l2', 'l1', 'l2'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'rank_test_score': array([14, 12, 13, 11, 10,  9,  2,  1,  4,  3,  5,  6,  7,  8]),\n",
       " 'split0_test_score': array([-0.69314718, -0.63852207, -0.67348477, -0.53085928, -0.51625497,\n",
       "        -0.52213248, -0.54365624, -0.54512894, -0.54940955, -0.54957849,\n",
       "        -0.55003341, -0.55006292, -0.55009414, -0.5501118 ]),\n",
       " 'split0_train_score': array([-0.69314718, -0.64155772, -0.67353923, -0.52027711, -0.46422351,\n",
       "        -0.4553345 , -0.44936869, -0.44922589, -0.4491079 , -0.44910642,\n",
       "        -0.44910513, -0.44910511, -0.4491051 , -0.44910509]),\n",
       " 'split1_test_score': array([-0.69314718, -0.64313232, -0.67019934, -0.53323282, -0.49258542,\n",
       "        -0.47504489, -0.46970169, -0.46791974, -0.46760249, -0.46744349,\n",
       "        -0.46741541, -0.4674009 , -0.46739825, -0.4673967 ]),\n",
       " 'split1_train_score': array([-0.69314718, -0.63938386, -0.6638873 , -0.52465661, -0.47971119,\n",
       "        -0.4732069 , -0.46942617, -0.46927278, -0.46920601, -0.46920442,\n",
       "        -0.4692037 , -0.46920368, -0.46920367, -0.46920367]),\n",
       " 'split2_test_score': array([-0.69314718, -0.64515276, -0.67205658, -0.53725593, -0.48460372,\n",
       "        -0.47950894, -0.47340242, -0.47251922, -0.47210127, -0.4720269 ,\n",
       "        -0.4719902 , -0.47198193, -0.47197777, -0.47197664]),\n",
       " 'split2_train_score': array([-0.69314718, -0.63826514, -0.6708104 , -0.52258289, -0.47847116,\n",
       "        -0.47014965, -0.46626205, -0.46611877, -0.46605052, -0.46604908,\n",
       "        -0.46604833, -0.46604832, -0.46604831, -0.46604831]),\n",
       " 'split3_test_score': array([-0.69314718, -0.64528241, -0.67784717, -0.54129636, -0.48888116,\n",
       "        -0.49143606, -0.48694541, -0.4874715 , -0.48734821, -0.48741468,\n",
       "        -0.48740794, -0.48741549, -0.48741516, -0.48741564]),\n",
       " 'split3_train_score': array([-0.69314718, -0.63803941, -0.68048941, -0.52063778, -0.47526118,\n",
       "        -0.46672304, -0.46264797, -0.46250653, -0.46243418, -0.46243276,\n",
       "        -0.46243199, -0.46243197, -0.46243196, -0.46243196]),\n",
       " 'split4_test_score': array([-0.69314718, -0.63492438, -0.67401484, -0.5027844 , -0.44348144,\n",
       "        -0.43159754, -0.42135406, -0.42090136, -0.41992297, -0.41988807,\n",
       "        -0.41979355, -0.41978853, -0.41977514, -0.4197786 ]),\n",
       " 'split4_train_score': array([-0.69314718, -0.64348716, -0.67531067, -0.53360682, -0.49173018,\n",
       "        -0.48285516, -0.47918002, -0.47904276, -0.47897923, -0.47897783,\n",
       "        -0.47897715, -0.47897713, -0.47897713, -0.47897713]),\n",
       " 'std_fit_time': array([0.00074835, 0.00040002, 0.00048994, 0.00048994, 0.00048986,\n",
       "        0.        , 0.        , 0.00623999, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.00040007]),\n",
       " 'std_score_time': array([1.78416128e-07, 7.48442917e-04, 1.16800773e-07, 4.89901382e-04,\n",
       "        4.89920847e-04, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "        6.23998642e-03, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "        6.23998642e-03, 6.24008179e-03]),\n",
       " 'std_test_score': array([0.        , 0.00405237, 0.00253123, 0.0135838 , 0.02354291,\n",
       "        0.02923962, 0.03925136, 0.04001434, 0.04175981, 0.04184109,\n",
       "        0.04202454, 0.04203738, 0.04205202, 0.04205718]),\n",
       " 'std_train_score': array([0.        , 0.00208387, 0.00546574, 0.00488415, 0.00881916,\n",
       "        0.00895297, 0.00970995, 0.00971056, 0.00972866, 0.00972867,\n",
       "        0.00972888, 0.00972888, 0.00972888, 0.00972888])}"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view the complete results (list of named tuples)\n",
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.47897787907349854\n",
      "{'C': 1, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如果最佳值在候选参数的边缘，最好再尝试更大的候选参数或更小的候选参数，直到找到拐点。\n",
    "l2, c=100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VNX5+PHPM5MNSAKBBAiEfZEt\nJEjYBRRBEBVXUH/iVi3VL35b24LiUlu1WnDf2oKiiF+puBFARGRRRLGiiCwJyCLIGpIQyErWmfP7\nYyYhQAKTZCZ3kjzvvqZzl3PvfS5Inpxz7j1HjDEopZRSNWWzOgCllFL1gyYUpZRSXqEJRSmllFdo\nQlFKKeUVmlCUUkp5hSYUpZRSXqEJRSmllFdoQlFKKeUVmlCUUkp5RYDVAdSmyMhI07FjR6vDUEqp\nOuXHH388ZoyJOl+5BpVQOnbsyMaNG60OQyml6hQR2e9JOW3yUkop5RWaUJRSSnmFJhSllFJeoQlF\nKaWUV2hCUUop5RWaUJRSSnmFJhSllFJeoQlFKaWUV2hCUUp5zaB51zNo3vVWh+EV9eVeavM+NKEo\nZbE7V9zJnSvutDoMpWqsQQ29opQ/2p6SbXUISnmF1lCUUkp5hSYUpZRSXqEJRSmllFdYklBEpLmI\nrBKR3e7viHOUDReRwyLyWrlta0Vkp4hsdn9a1k7kSimlKmNVDWUGsMYY0w1Y416vzJPAVxVsv8UY\nE+/+pPkiSKWUUp6zKqFcDcx3L88HrqmokIj0B1oBK2spLqWUUtVkVUJpZYxJAXB/n9VkJSI24Hlg\neiXnmOdu7vqLiIjvQlVKKeUJn72HIiKrgdYV7HrEw1P8D7DcGHOwgnxxizHmsIiEAR8DtwLvVBLH\nFGAKQPv27T28tFJKqaryWUIxxoyubJ+IpIpItDEmRUSigYr6QIYAw0Xkf4BQIEhEco0xM4wxh93X\nyBGR/wADqSShGGNeB14HSEhIMDW7K+UvSt8snzdunsWRKKVKWdXktRS43b18O7DkzALGmFuMMe2N\nMR2BacA7xpgZIhIgIpEAIhIIXAkk1U7YSimlKmNVQpkJjBGR3cAY9zoikiAic89zbDDwuYhsBTYD\nh4E3fBmsUkqp87NkLC9jTAZwaQXbNwJ3V7D9beBt93Ie0N+3ESpVe/66YIdroR6MD6n34n9q8z70\nTXmllFJeoQlFKaWUV2hCUUop5RU6H4qqk256Ndm1MM7aOJRSp2hCUXXSySKH1SEopc6gCUUpqxmD\nGHDk5pVuOG3feZfLrxuDMebUOZxO17IxYJynlaO0nClXBgPO0nMbjNNZ7vym3PHOM8q79jW2uRJ9\n8a7N1fqj8Cf15V5K78MUnERCGvv0WppQlPIC43TizMvDkZWFIysLZ3Y2jqxs13r2GeuZx3FmpOLI\nysSRk0enAgMIuxISrL6NGmvl7pbdM+FmiyOpufpyL6X3UZT0HcEJo3x6LU0oSrkZY3DmncSZnYUj\nOxtHZgXJ4Kz1bJxZWThycty1gUrYbdgb2bAHOLDbC7EHOQlq7MQeFUJBQQHFRmjeNwEQ16f88HXi\n3gaI7dQyZWPclT9GXNvLj39Xvpyt3HJpeUDKjqng3HLGuc+Mq9y+Q6tcg17EjKlwAPE65dCqxUDd\nv5fS+who383n19KEouoVYwymoKAsIZQlh/IJISvbve2M9exsKCmp/OR2O/bwcOzh4diaNcUeEUFQ\nhw7Ym4ZjCw/HHlCC3XkCe3Eq9oJD2HL3YHdkYA8yiN0gLTpD637Qui9Ex7m+w1qxfHQvAMbP+qiW\n/pR8578rPgcgdtoLFkdSc/9dsQKo+/dSdh8t2/n8WppQVJ0UUOIkMruYg7+7pywZOLKycGZlYYqL\nKz9QxJUQmjYtSw6Bbdpgb9oUe3jTU8mh3LqrfDNsTRq7fpMvKYS0HXB0KxzdBinrITUJinJd17AF\nQsse0HsstI6F6L7Qqg+EhNfOH45SFtGEouqck5t+olNqAWKgOD0Ne9OmBLfs6koQTUuTRflk0NSd\nIMKxhYYitiq8flWQ7UoaSVshxZ1A0neA012TCQp1JY34W04lj6geEBDsm5tXyo9pQlF1SubixRz9\ny2M4BVKjYMyiRd45sTGQm+pOGltOJY8T+06VadLSlTC6jXF9t+4LEZ2gKglKqXpME4qqE4zTSfqL\nL5LxxlwaDxzIrwe/x2mv5kSdTqcrUaRscTVbpWx1feelnyoT0cmVNPpNdvd3xEJYRfPFKaVKaUJR\nfs+Zl8fhBx4kd80amk2aROu/PEry5XGeHVzW37HtVPI4rb8jAKJ6QrfL3J3l2t+hVHVpQlF+rfjw\nYQ7+z1QKd++m1cMPE3HrZCqYEtqltL+jrLN8K6T/DE53J31QqCtZxP+/U8nDD/o7Hr+lJwDjLY3C\nO/Re/E9t3ocmFOW3Tv70E4fu+19MURHt5swhdPhFp+1vHl4I6549T3/H6FOP6Wp/h1I+pQlF+aWs\nJUtIefQvBERH0+7/3iG4c+fT9kdFFDCgdwZ88ffT+ztKax5e6u8wxuA0ThzGQYmzhBJTgsPpOLXu\nLMFhHDicDoqdxWXLJeb0fRUdW+QoJrugkIKSIjB2/vHNm9hEsIkNm9gQyi2LYEOwif20Mqcvn70u\nAjbs2GyCDRt2sWGz2bBR+i2ub3HvE9c17CKua4ode+mxtlPXsIsdEZtrX9mxNpxOV+0xMz/3jD/J\n02uVpbXM0sqmlNsv7v+dXv6sU2A7swxyWu217Brl9p92vnJlzlxW1SPmzDGB6rGEhASzceNGq8NQ\n5+DqfH+JjDfeoPHAgbR9+SUCIiLOKvfJszEkBwdQ1O8WHPaAs364V/jD371eWu6074qSgHtdNVzG\n1J8k89qI/3Bx5z7VOlZEfjTGnHdsIK2hKL9RUee7BAaeVe6b/V/wTItwcmw2mh1Zj91mJ0ACXN+2\nAOzi+i7dVroeYgvBLvay8gG2U/sDbYFl+8rWyx1b4hAKig35RZBfZDhZaMgrMOQVOsktMOQUOMjJ\nN2TnO3E4BbCBsYGxY7ARaAsgolEIEU1CaNG4Ec2bhBDZpBGRoY2YvfktbLYiZo56oFyNyInTOHEa\ng9M4cGLK9p3a7sQYJ04MTqfDtY1T+5zGiaG0nHGXdeJwGgynzmOM071u3NdxnjoGc6ocDnd513WM\n+/zl17856PqFbWjMhWV/X6bc/7uWTPkdnP0rrTljrYISZ/0iXFGp8mNolr9+BUvm7Ov+lOqaIiG+\nVa8Kzlx3bE7dDkC7ppE+v5YmFOUXio8ccXW+79pVaed7ibOE1356jTeT3qSrw8GQkw6euWNtta95\nsqiEYzlFpOcWkJ5TSHpuEenZhaTnFHIs1/VdulxYcvY4XQE2ITI0mMiwIDqGBhMVFUxkaDBRYa5P\n+eWw4IBKm1Tm73kagHE9fT/Wkq8Nmnc9AK9fPd3iSGqu9F7evf6vFkdSM6X30aWF7x9714SiLFfW\n+V5YSLs5swkdPvysMkfzjvLgugfZlLaJ6wMi+Z8DW/h9BWMTFZY4OJZbxDF3MkjPLXQt556dKPIq\nmFNFBFo0CSpLBp0jm5yVHEqXmzUKxGarP00iStWUJhRlqaylS0l55FFX5/s78wnu0uWsMusOreOR\nbx6hyFHEzCFPcMWHU1kZFMr+42P53/d+Ij2ngGO5RaTnFJKVX/E4Xk0bBboSQmgwsTHNiAotTQ5B\nZYkiKjSY5k2CCLDrk2BKVYcmFGUJ43SS/tLLZLz+eqWd78XOYl7d9CrzkufRPaI7z418jk6Ht0Jx\nHivsF5OSNZKthzJpGRZMt5ahDO3SolyiOFWjaBEaRHCA3aI7Varh0ISiap0zL4/DDz5I7uo1NJs4\n0dX5HhR0WpmU3BSmr5vOlvQtTOw+kQcGPEBIQAisfBwaR7L85HjsQSdYO228Pu6plJ/QhKJq1emd\n7w8RceutZyWEtQfX8uj6Ryl2FPPMiGe4vNPlrh1FebDrc/J63EDBD21p0mKTJhOl/IgmFFVrztf5\nXuws5uUfX2b+9vn0aN6D50Y+R4fwDqcK7Pocik/yRcAwQAgJ31u7N6CUOidNKKpWlHW+t25dYef7\nkdwjTP9qOluPbeXGC25k+oDpBNvPGGMrORGatOSN/a0JCPmVgKCcWrwDpdT5aEJRPuVJ5/sXB77g\n0fWP4jROnh35LOM6jjv7RIW5sHslWT1vYusPeYRGae1EKX+jCUX5zPk634sdxbzw4wu8u+Ndejbv\nyXMjn6N9ePuKT7ZrBZQUsJIhiKDNXUr5IU0oyifO1/l+KOcQ07+aTlJGEjf3uJlpCdMIsgdVfsLk\nRExoa+bsjWJQp8bsC8ivhbtQSlWFJW9wiUhzEVklIrvd32eP/ucq5xCRze7P0nLbO4nIBvfx74vI\nOX4Sqdp28qef2DfpRooPHaLdnNk0v+2205LJmv1rmPTJJH7N/pUXLn6Bhwc9fO5kUpgDu1dxvMM4\n9mQUMCGubS3chVKqqqx6JXgGsMYY0w1Y416vSL4xJt79mVBu+yzgRffxJ4C7fBuu8lTWJ59w4PY7\nsDVqRMf3F572JFeRo4iZ38/k/rX30z68PR9c9QFjOow5/0l3fgaOQj51DibQLlzeR6fiVcofWZVQ\nrgbmu5fnA9d4eqC4ftUdBXxUneOVbxink7QXX+LI9AdoFBdHxw/eP+1JroM5B7n1s1tZsGMBk3tO\n5p3L36Fd2NljcVUoORET1obZv0QyolsUEU20QqqUP7IqobQyxqQAuL9bVlIuREQ2ish3IlKaNFoA\nmcaY0okqDgHaBmIhZ14eh//wBzLmzKHZxIm0f3PuaU9yrfx1JZM+mcTBnIO8dMlLPDjwwXM3cZVX\nkAV7VnM0ZixHsouYEN/GR3ehlKopn3XKi8hqoKK2iUeqcJr2xpgjItIZ+EJEtgHZFZSrdJYwEZkC\nTAFo376SJ4hUtRWnpLg633fupNVDM4go119S6CjkuR+eY+HOhcRGxvLsyGdpG1rF3L/zM3AUsaR4\nECGBNkb3bOWDu1BKeYPPEooxZnRl+0QkVUSijTEpIhINpFVyjiPu770ishboB3wMNBORAHctJQY4\nco44XgdeB9eMjdW9H3W2/M2bOXjf/2IKCmg3+9+EjhhRtu9A9gGmfTWNHcd3cFuv27j/wvsJtJ89\nWdZ5JSdiwtvy+i8RjO7ZkibB+mCiUv7KqiavpcDt7uXbgSVnFhCRCBEJdi9HAsOA7cY1VduXwA3n\nOl75VtYny9h/2+2uzveF752WTFbsW8GkZZM4nHuYVy55hekDplcvmeRnwp41HIwey/F8B1fHa8um\nUv7Mql/3ZgIfiMhdwAFgIoCIJAD3GGPuBnoCc0TEiSvxzTTGbHcf/yCwUET+DvwEvFnbN9BQGaeT\n9JdfIWPOHBoPGEDbV14u6y8pdBTyzPfP8MGuD+gb1ZdnRzxLm9Aa9HnsXA7OYhYVDiA8JIAR3X0/\nhalSqvosSSjGmAzg0gq2bwTudi9/C8RWcvxeYKAvY1Rnc548yZEHHyRn1WqaTbyB1n/5S9mb779m\n/cq0r6ax88RO7uh9B7+/8PcE2qpRKykvaRHOpu14Y28EV/aN1jlNlPJz2iCtPHKuzvfle5fz+H8f\nJ9AeyD8v/ScjYkac52weOHkc9n7Jvi63k5fq1Ke7lKoDNKGo88rfsoWDU+87q/O9oKSAWT/M4qNd\nHxEfFc+zI5+ldRMvvXT486fgLOH9k/2JCgtmcOcWp+3uFR3unesopbxGE4o6p6xPlpHyyCMEtGpF\nu7fnEdy1KwD7svYx7atp7Dqxi9/0+Q339buv5k1c5SUn4mzagbf3R3DLoGjsttMn0po3bp73rqWU\n8gpNKKpCxukk/ZVXyJh9duf7sr3LeOK/TxBsD/ZeE1d5J4/D3rXs7HwHRamGCXHa3KVUXaAJRZ3F\n1fk+g5xVq07rfM8vyWfm9zNZtHsRF7a8kFkjZnmviau8HZ+AcbAg90LaN29MfLtm3r+GUsrrNKGo\n0xSnpHBw6lQKfz69831v5l7+/NWf2ZO5h7tj72Zq/FQCbD76zyd5ESXNOvGfA8249+JonTdeqTpC\nE4oqU9b5np9/Wuf70l+W8vfv/k6IPYTZo2czrO0w3wWRdwz2rWN7x9/gPCr6MqNSdYgmlAbmzhV3\nAmd3apd1vrdsSbt5bxHcrRsni0/y9IanWfLLEvq36s+s4bNo1cTHY2ntWArGyduZ8fRoHUb3VmG+\nvZ5Syms0oTRwp3W+JyTQ9tVXCIiIYM+JPUz7ahp7s/Yype8U7o2713dNXOUlJ1LcrAuLUiKYPlY7\n45WqSzShNGDlO9+b3nA90Y89hgQFsXjPYp767ikaBzZm9pjZDG0ztHYCyk2DX79hS/s74ag0mKe7\nNtz5sdUhKOUVmlAamJteTQagOO5U53vLGQ/S/PbbyS/J56lvHmfpL0sZ0HoAs4bPIqpxVO0F527u\nmns8ngvbN6Nd88a1d22lVI1pQmmAggoc7Js0CXMyn3b//hehI0ey+8Ru/vzVn/k161fujbuX3/X9\nHXZbLY+dlZRIYUQ3VqQ0529XNYzaSX1Tn2pb9eVeavM+NKE0MI1zimmRXoCtbXPavfUWQV27smj3\nIp7e8DShgaG8ftnrDI4eXPuB5RyF/ev5MeYubCJc0VcTilJ1jSaUBqRwzx5apBVQGGKn+4cfUBQa\nzEPfPMSnez9lUPQgZg6fSWQji4aI374UMMw+1pehXSKJCgu2Jg6lVLVpQmlAUmc9g7HBsdYh/GLS\nmLZsGgdyDjA1fiq/jf1t7TdxlZecSH6z7qw7Gskzl2rtRKm6SBNKA5H79dfkff01mS2C+Labk4eW\n30JYUBhzL5vLgNYDrA0u+wgc+C8bou8iyG5jbB8fDOeilPI5TSgNgCkpIXXWLALbt2PeoFQ2dC1m\nSMshPD38aeuauMpzN3e9lh7LxRdE0bSRF0ctVkrVGqvmlFe16MQHH1C05xcO33YpG7o6ad24NbPH\nzPaPZAKQvIi8Zj3YmBulE2kpVYdpQqnnHNnZHHvlVYIH9OfJoJU0DmhM29C22MRP/uqzDsHBDXwT\nPJwmQXYu7eHjoV2UUj7j0U8VERkmIk3cy5NF5AUR6eDb0JQ3HPv3bBxZWXx1XRdS89NoF9bOv0bv\n3b4EgH+m9uGy3q1pFKTzxitVV3n6a+q/gZMiEgc8AOwH3vFZVMorivbv5/i77xJ41Vhey13G+E7j\nCQvys8EWkxPJbtaTrQVRDWaoFaXqK08TSokxxgBXAy8bY14G/OwnkzpT2nPPIYGBvDW0ALvNzh/7\n/9HqkE6XeQAO/cDagIuIaBzIRd38pE9HKVUtniaUHBF5CJgMfCoidkAfxfFjeRu+J2fVak7eNJZP\nsr7ht7G/9c3sijXhbu56LbUPl8dGE2j3k34dpVS1ePov+EagELjLGHMUaAs867OoVI0Yh4PUWTMJ\niI7m6U5JxITGcFvv26wO62xJi8hs1ptdxdrcpVR94HENBVdT19ci0h2IB97zXViqJrIWL6Fw+w52\n3TSInSf3MX3AdILtfjaUyYlf4cgmVstQWoeHMLBjc6sjUkrVkKcJZR0QLCJtgTXAncDbvgpKVZ8z\nL4+0l14ksG9vngpdy9A2Q7mk3SVWh3W25MUA/DOtD1fFRWOz+dGTZ0qpavE0oYgx5iRwHfCqMeZa\noLfvwlLVdWzuXBzpx/jsqtbkOwp4cMCD/vWYcKnkRDKa9mGfI4oJcTpvvFL1gccJRUSGALcAn7q3\n6QsDfqb4yBGOvzUPM+Yi3nCu4+aeN9O5WWerwzrb8b2QspnPzGA6RTahT9twqyNSSnmBp2N53Q88\nBCQaY5JFpDPwpe/CUtWR9vwLAPxzcBYRARHcE3fPWWXmjZtX22Gdzd3cNTs9lutHtfHPGpRSqso8\nSijGmK+Ar0QkTERCjTF7gd/7NjRVFfmbN5P96aecuPFS1pV8xeMDHyc8yE9/809OJDU8lkNpOnaX\nUvWJp0OvxIrIT0ASsF1EfhSRavehiEhzEVklIrvd3xGVlHOIyGb3Z2m57W+LyL5y++KrG0t9YIwh\n9R8zsUe24MmuyfRu0Ztrul5jdVgVy/gFjm5lmWMQvduE0yUq1OqIlFJe4mkfyhzgT8aYDsaY9sCf\ngTdqcN0ZwBpjTDdcT43NqKRcvjEm3v2ZcMa+6eX2ba5BLHVe9qfLyd+yhS3XxXLIeYwZA2f4z+CP\nZ0peBMDcjDh990SpesbTnzpNjDFlfSbGmLVAkxpc92pgvnt5PuCnv077P2dBAWnPP4+texdmtviO\nqzpfRXxLP66wJS/mSHhfUmjBlZpQlKpXPE0oe0XkLyLS0f15FNhXg+u2MsakALi/W1ZSLkRENorI\ndyJyZtJ5SkS2isiLIlLpW3siMsV9jo3p6ek1CNk/HX/7bUpSUlg0vin2gEDu73+/1SFVLn0XpCaR\nWDiQgR2b07ZZI6sjUkp5kacJ5TdAFLAISHQv33muA0RktYgkVfC5ugrxtTfGJAD/D3hJRLq4tz8E\n9AAGAM2BBys7gTHmdWNMgjEmISoqqgqX9n/FaWkce/0Niob14z+NtvK7vr+jZePKcrMf2L4Yg/BO\nVjxXaWe8UvWOp095naCKT3UZY0ZXtk9EUkUk2hiTIiLRQFol5zji/t4rImuBfsAvpbUboFBE5gHT\nqhJbfZH+8suYoiJeHJxB+7D23NrrVqtDOrfkRA6FxXGsqAXjdd54peqdcyYUEfkEMJXtr6Cj3FNL\ngduBme7vJRVcOwI4aYwpFJFIYBjwjHtfaTISXP0vSdWMo84q2L6drEWJpFyZwI9BP/HagNcIsgdZ\nHVbl0n6GtO18GPBbLuoaSYtQPxtbTClVY+eroTzno+vOBD4QkbuAA8BEABFJAO4xxtwN9ATmiIgT\nV9PcTGPMdvfxC0QkChBgM3D2G3z1mDGG1JmzkKbh/L3nTi5qexEjYkZYHda5JSdiEN7Ljechbe5S\nql46Z0Jxv9DodcaYDODSCrZvBO52L38LxFZy/ChfxFVX5K5Zw8nvv+eHW/pxImAHDwx4wL/fNjcG\nkhP5NTSe7JIWXNZbm7uUqo886kMRkW2c3fSVBWwE/u5OEKoWmKIiUp99FtMxhhditnFrrzvo1LST\n1WGdW9oOOLaT9213c2nPloQGezrij1KqLvH0X/ZngAP4j3v9JlzNTVm4hrG/yuuRqQodX/Afivcf\n4L3fdqFZ4xb8ru/vrA7p/JITMWLjo5P9+bu+e6JUveVpQhlmjBlWbn2biKw3xgwTkcm+CEydreTE\nCY7961/k9b+AxZG/8GT/JwkN8vOhS9zNXXsax1PobMHFF/jxY81KqRrx9D2UUBEZVLoiIgOB0p9k\nJV6PSlXo2Kuv4Tx5kueHZBAbGcuELtV9yK4WpSZBxm7ey+3P2D6tCQnUWQ+Uqq88raHcDbwlIqG4\nmrqygbtEpAnwD18Fp04p3LOHE++/z/5LLiApbBcLBv7Tf8frKi85EYONxYX9eUmbu5Sq1zx9sfEH\nIFZEmuKavTGz3O4PfBKZOk3qM89AoxCejv2Fq7tcTd+ovlaHdH7u5q6fG/XDFhDJ0C4trI5IKeVD\nng5f31REXsA1MvBqEXnenVxULcj9+hvy1n3NN2OiKQoL8e/xuso7uhWO72VB7oWMj40mwF4HalRK\nqWrz9F/4W0AOMMn9yQb8YOq/+s+UlJA6ayaONlH8q+s+7ul7D5GNIq0OyzPJiTjFzrLiBK7WlxmV\nqvc87UPpYoy5vtz64yLSoOcgqS2ZH35I0Z5feHdyK2Kad+KWnrdYHZJn3M1dScH9aBLckgvbVziH\nmlKqHvG0hpIvIheVrojIMCDfNyGpUo7sbNJffoXsXu34NOYYDwx4gEB7oNVheebIT3DiVxbkXshV\ncTpvvFINgac1lHuB+aWd8sBx4A5fBaVcjv17No6sLJ67oZiR7S5meMxwq0PyXHIiTglgRUkC7+nT\nXUo1CJ4+5bUZiBORcPd6tk+jUhTt38/xd99lz9D27GmVynMDplsdkueMgeTFbA7sR8uw1vSMDrM6\nIqVULTjf8PV/qmQ7AMaYF3wQkwLSnnsOE2DjmQsPcVuvu+gQ3sHqkDx3eBNkHeA/xZczYZA2dynV\nUJyvhqK/Wlogb8P35KxazdpxrQmMEqb0nWJ1SFWTvAiHBLDSkcBSbe5SqsE43/D1j9dWIMrFOByk\nzppJcVRT5vZJ5/H+/6BJYBOrw/Kcu7lrY0A/OsW0oWNkHYpdKVUjVX7TTEQ2+SIQ5ZK1eAmF23fw\nfyMMvdrEc0XnK6wOqWoObYTsQyzMS+AqrZ0o1aBU59VlbRD3EWdeHmkvvciJrlF83u0kDw18qG6M\n11Ve8iJKJJA1pr8mFKUamOr8tPrU61EoAI7NnYsj/RgvDsvi2u7X0Tuyt9UhVY3TiUlezAZbP3p3\naker8BCrI1JK1aIqJxRjzKO+CKShKz5yhONvzWNn/5Yc6tCY3/f7vdUhVd2h75GcI3yQn8AEHWpF\nqQbH08Ehc0Qk+4zPQRFJFJHOvg6yIUh74UWcxslLgzK4N+5eWjSqgyPzJidSIkF8JQlc3kfnjVeq\nofH0TfkXgCO4pgAWXFMAtwZ24ho48mJfBNdQ5G/ZQvayZay+uCnh7Vtyc8+brQ6p6tzNXd/Qj4Tu\n7WnWOMjqiJRStczTJq9xxpg5xpgcY0y2MeZ1YLwx5n1AR/2rAWMMqf+YSVGzJrzTP5cZA2YQaKsj\n43WVd/A7JPcoHxcO0M54pRooTxOKU0QmiYjN/ZlUbp/xRWANRfby5eRv3sw7wx0M7TKKoW2HWh1S\n9SQtoliC+NY+gDG9WlkdjVLKAp4mlFuAW4E0INW9PFlEGgH3+Si2es9ZUEDa889zvF1T1vaB6Ql1\naLyu8pwOzPYlfGX6MbRXBxoHedqSqpSqTzwdHHIvcFUlu7/xXjgNy/G351NyJIVX/p+N22N/R7vw\ndlaHVD37v0Xy0kgsuolrtLlLqQbL06e8uovIGhFJcq/3FRF9fLgGStLTOfb66+zoE86xHq25O/Zu\nq0OqvuRECiWEH4MGMLJ7lNXRKKUs4mmT1xvAQ0AxgDFmK64nvVQ1pb38Ms6iQv41LI8/JfyJxoGN\nrQ6pehwlmO1L+cLZj4tjOxIZT1cbAAAc0UlEQVQUUMfe7FdKeY2n//obG2O+P2NbibeDaSgKduwg\n6+NFrB4QRJseFzK+03irQ6q+/euRk+ksKR7EBG3uUqpB87T39JiIdMH9RJeI3ACk+CyqeqzsMeEm\nQfxncDFvDZxRt+cLSV5EgYSQ3HgggzrXwZcxlVJe42kNZSowB+ghIoeB+4F7qntREWkuIqtEZLf7\nu8J3WUSkvYisFJEdIrJdRDq6t3cSkQ3u498XkTrzFl3umjWc/P57Fgx1cHnfG+jVopfVIVWfowTn\n9qWsdlzImLhO2G11ODEqpWrM04RyGJgHPAUsBFYBt9fgujOANcaYbsAa93pF3gGeNcb0BAbiemwZ\nYBbwovv4E8BdNYil1piiIlKfeZaMVo34dmAo/9vvf60OqWZ+XYct/ziflAzSsbuUUh4nlCW4Hhsu\nxjUESy6QV4PrXg3Mdy/PB645s4CI9AICjDGrAIwxucaYk+JqHxoFfHSu4/3R8QX/ofjAAeaMLOTe\n/vfRPKS51SHVTHIi+dKIvc2GEBfT1OpolFIW87QPJcYYM86L121ljEkBMMakiEjLCsp0BzJFZBHQ\nCViNqyYTAWQaY0ofCjgEtPVibD5RcuIEx/71T3Z0CyHnwg5MumDS+Q/yZ45inNs/4fOSCxkb17Fu\n9wMppbzC04TyrYjEGmO2eXpiEVmNawDJMz3i4SkCgOFAP+AA8D5wB7C0grKVDv8iIlOAKQDt27f3\n8NLed+zV13Dk5fHGSBuPDXywbo7XVd6+r7AVnGCZYzAPaHOXUgrPE8pFwB0isg8oxDXisDHG9K3s\nAGPM6Mr2iUiqiES7ayfRnOobKe8Q8JP7LX1EZDEwGNfoxs1EJMBdS4nB1QxXWRyvA68DJCQkWDLu\nWOGePZx4fyFrLgzggv6jGNJmiBVheFdSInnSmNSooXRvFWZ1NEopP+BpQrncy9ddiqtTf6b7e0kF\nZX4AIkQkyhiTjqvfZKMxxojIl8ANuB4QqOx4v5H6zDMUBdn4aHgACxKmWR1OzZUU4dzxCStKLuTy\nfh2tjkYp5Sc86pQ3xuyv6FOD684ExojIbmCMex0RSRCRue5rOoBpwBoR2YarVvSG+/gHgT+JyB6g\nBfBmDWLxqdyvvyFv3dcsHOJk4sDfEBMWY3VINbd3LbbCLD51DOaqvtrcpZRysWRYWGNMBnBpBds3\nAneXW18FnNWs5m4GG+jLGL3BlJSQOnMmGS2C+Gl4JH/p8xurQ/KO5ERyaUJu2+G0a15Hh4xRSnmd\nDrzkQ5kffkjRL7/w1sgS7h88ve6O11VeSSGOHcv4rKQ/V2hzl1KqHE0oPuLIzibtlVfY2SEA5/AE\nxnYca3VI3vHLl9iLslluBjM+NtrqaJRSfkQTio8cmz0HR2Ym8y6FGYMeqjfvaZjkRWQTiuk0kqiw\nYKvDUUr5EU0oPlB04ADH33mHr2JtDBhxIz2a97A6JO8oLsC541M+K0lgfHwHq6NRSvkZTSg+kPrs\ncxTZnHwyOpz74uvRDMm/rMFenMsKhjC2d0XvrCqlGjJNKF6W9/335K5axaLBcOuI39MspJnVIXmN\nMymRTMII6X4xTRvV8Tf9lVJepwnFi4zTydGZMznR1M6Osd2Y2H2i1SF5T3E+5uflLC9J4Ept7lJK\nVUATihdlLV5C0fYdvDPSMG3owwTYLHnNxzf2rMZeksca21Au7VnRWJ5KqYauHv3Es5YzL4+jLzzP\nnrY2Gl9+GQOj/f69yypxbFtEFmE06zWKkEC71eEopfyQJhQvyXjzTcyxDBbcEcILA6ZbHY53FZ3E\n7FzBZyVDuLKfdSM2K1UTxcXFHDp0iIKCAqtD8VshISHExMQQGFi9PlJNKF5QnJJC+ptv8m1PYeS4\nKbQJrWfjW+1eSYDjJGsDL+JfXSOtjkapajl06BBhYWF07Kjz91TEGENGRgaHDh2iU6dO1TqH9qF4\nQerzz1PiKGbV+Nbc0ecOq8PxupJtizhmmtIqdhSBdv1PRtVNBQUFtGjRQpNJJUSEFi1a1KgGpz8d\naih/yxZyln3KJwPhrjEzaBTQyOqQvKsoD3Z/znLHQK6Kb2d1NErVSFWTyY1z/suNc/7ro2j8T02T\nrSaUGjDGcPjpp8gMFQ5cPYAxHcZYHZL37fqcAEcB3zUawYCOza2ORqk6LTQ0tGx53LhxNGvWjCuv\nvLLCslOnTiU+Pp5evXrRqFEj4uPjiY+P56OPPqrSNTdt2sSKFStqFLentA+lBrKXL6d4yzYWXmHn\nzyMeqZdV6aKtH5NpmtEufhQ2W/27P6WsMn36dE6ePMmcOXMq3P/Pf/4TgF9//ZUrr7ySzZs3V+s6\nmzZtIikpiXHjxlU7Vk9pDaWanAUFHHlmFvtaCS2vv5HuEd2tDsn7CnOw7VmlzV1K+cCll15KWFj1\nps/evXs3Y8eOpX///owYMYJdu3YBsHDhQvr06UNcXByXXHIJ+fn5PPHEEyxYsKBatZuq0hpKNWW8\n/TakpvPR7eG82P/3VofjG7s+J8BZyKawS7i9TbjV0SjlNY9/ksz2I9nnLbc9xVXGk36UXm3C+etV\nvWscmyemTJnC3Llz6dKlC+vXr+e+++5j5cqVPP7446xdu5ZWrVqRmZlJo0aNeOyxx0hKSuKll17y\neVyaUKqhJD2dtDmz2dhdGHfdn2ka3NTqkHyiYPNHZJoIOvUbVS+b85SqizIzM/nuu++4/vrry7aV\nlJQAMGzYMG677TYmTpzIddddV+uxaUKphpQXX8BZVMj6a7ry727Xn/+Auqggm4B9a1juGMWEfjFW\nR6OUV3lakyitmbz/uyG+DKdKjDFERkZW2KfyxhtvsGHDBpYtW0ZcXBxbt26t1di0D6WKCnbsICdx\nMZ/1F6aM/yt2Wz0dhmTXCgKcRWxvfildokLPX14pVSsiIiKIjo4mMTERAKfTyZYtWwDYu3cvgwcP\n5sknnyQiIoLDhw8TFhZGTk5OrcSmCaUKjDEceOoJckMg6+bLSGidYHVIPpP304ccMc3p3v8Sq0NR\nql4aPnw4EydOZM2aNcTExPD55597fOzChQuZPXs2cXFx9O7dm2XLlgHwxz/+kdjYWGJjYxk9ejR9\n+vRh1KhRbNmyhX79+mmnvD/J/eILHBs3s2hcEPePmGF1OL6Tn0nIr1/ynmM0V8Zpc5dS3pKbm1u2\n/PXXX3t0TMeOHUlKSjptW+fOnStMQEuXLj1rW1RUFBs3bqxipNWjCcVDpqiIA08/yZEW0Om2e2jd\npP7OWGh2LsduitnbcixtmtWzN/+VqgJ/6jupC7TJy0PHFryL7XAqn17Rktv73ml1OD6Vu+lDDplI\neg3Q5i6llOe0huKBVeMG0PJQLts7CVfd8hghASFWh+Q7+SdofHAdC53juL5vPRs1WSnlU1pD8UDj\nrELsDth8Uzyj2o+yOhyfcu74FLsp4UjbcTRvEmR1OEqpOkRrKB7Y3tbJiW7Cbyc8Ue9f8Mv+8UNy\nnFHEDrjY6lCUUnWM1lA8sLddIEeig+ka0dXqUHzr5HHCDn/NCgZzWZ9oq6NRynrzrnB9lEc0oXjg\n6Nh4si6JtzoMn3Ns/wQ7DjI6XElosFZelfK22h6+PjExkWeffbbGcXtKf2qoMlkbPyDb2Yr4gSOt\nDkWpes9bw9eXlJQQEFDxj/Jrr73WO8F6yJIaiog0F5FVIrLb/R1RSbn2IrJSRHaIyHYR6eje/raI\n7BORze6PT6sP88bNY964eb68hPXyMmh69L+skiFc3KOl1dEoVe/VZPj6iy66iEceeYQRI0bw2muv\nsWTJEgYNGkS/fv247LLLSEtLA2Du3Lncf//9AEyePJk//OEPDB06lM6dO5cN3eJNVtVQZgBrjDEz\nRWSGe/3BCsq9AzxljFklIqGAs9y+6cYY344j0IAUJy8hEAc5Xa4iJLCejk+mVKnPZsDRbecvd9Q9\nuKIn/SitY+HymTWLqwqys7NZt24dACdOnGDChAmICLNnz+b5559n1qxZZx2TlpbG+vXr2bZtG5Mm\nTfJ6DcaqhHI1cLF7eT6wljMSioj0AgKMMasAjDG5KJ/J2vgh2c7WDBg8wupQlFIeuOmmm8qWDxw4\nwKRJkzh69CiFhYV0717xhH/XXHMNIkLfvn05fPiw12OyKqG0MsakABhjUkSkojaW7kCmiCwCOgGr\ngRnGGId7/1Mi8hiwxr29sDYCr5fyjtE87TuW2K/l9i6RVkejlO95WpMorZnc+anvYqmmJk2alC1P\nnTqVhx9+mPHjx7N69Wpmzqz4/oKDg8uWjTFej8lnfSgislpEkir4XO3hKQKA4cA0YADQGbjDve8h\noId7e3Mqbi4rjWOKiGwUkY3p6enVvZ16rWDrImw4KbjgagLs+uCfUnVNVlYWbdu2xRjD/PnzLYvD\nZz89jDGjjTF9KvgsAVJFJBrA/Z1WwSkOAT8ZY/YaY0qAxcCF7nOnGJdCYB4w8BxxvG6MSTDGJERF\nRXn7NuuF7I0fssfZhsGDL7I6FKUajJoMX3+mv/3tb1x77bWMHDmSVq1aeTHKqrGqyWspcDsw0/29\npIIyPwARIhJljEkHRgEbwZWE3E1lAlwDJFVwvPJETiqRGRtZFnQDd3ZobnU0StVr3hq+/ptvvjlt\n/frrrz9tSuBSd999d9nyu+++W2ks3mJVQpkJfCAidwEHgIkAIpIA3GOMudsY4xCRacAad+L4EXjD\nffwCEYkCBNgM3FPrd1BP5G5eRChOTK9r6/2wMkpVmR/2nfgzSxKKMSYDuLSC7RuBu8utrwL6VlCu\nfo/QWItyN33EEWdbhg7R5i6lVM1oD2xDlnOUlid+5L+NRtCjdfVesFJKqVKaUBqwzB8/xIbB3uc6\nbe5SStWYjuXVgOX/9BEpznYMHzrM6lCUUvWA1lAaquwjRGdt5sfQi+nQosn5yyvVAN254k7uXFG/\np/z2Jk0oDVT6hvcBCIk7+1FDpZRvlA5fv3nzZoYMGULv3r3p27cv77///lllvTF8PcCmTZtYsWKF\nV+I/H23yaqCKti5iu7MDw4cMsToUpRqcxo0b884779CtWzeOHDlC//79GTt2LM2aNSsr4+nw9eez\nadMmkpKSGDdunFdiPxetoTRAJvMgbXO2sq3ZJbQKD7E6HKUanO7du9OtWzcA2rRpQ8uWLanK0FC7\nd+9m7Nix9O/fnxEjRrBr1y4AFi5cSJ8+fYiLi+OSSy4hPz+fJ554ggULFlSrdlNVWkNpgFL+u5A2\nQJN+E60ORSlLzPp+Fj8f//m85UrLeNKP0qN5Dx4cWOmwgpX6/vvvKSoqokuXLh4fM2XKFObOnUuX\nLl1Yv3499913HytXruTxxx9n7dq1tGrViszMTBo1asRjjz1GUlISL730UpVjqypNKA3NvCtwHviF\nJNOJ4YMGWR2NUg1aSkoKt956K/Pnz8dm86zBKDMzk+++++60oVZKSkoAGDZsGLfddhsTJ07kuuuu\n80nM56IJpYFxFBcSY1L4rvnd9GkcaHU4SlnC05pEac3EFzO2Zmdnc8UVV/D3v/+dwYMHe3ycMYbI\nyMgK+1TeeOMNNmzYwLJly4iLi2Pr1q3eDPm8tA+lgTmY5Zo2pmmCNncpZZWioiKuvfbastpEVURE\nRBAdHV02ha/T6WTLli0A7N27l8GDB/Pkk08SERHB4cOHCQsLIycnx+v3UBFNKA2FMXD4R4JyD7PN\n2YlhAxKsjkipBuuDDz5g3bp1vP3222WPA1flKa6FCxcye/Zs4uLi6N27N8uWLQPgj3/8I7GxscTG\nxjJ69Gj69OnDqFGj2LJlC/369dNOeVUDJUXw6zryt32C2fkZjQtSaYWwxFxEbJD+1StV20qHjJ88\neTKTJ0/26JiKhq/v3LlzhfOnLF269KxtUVFRbNy4sRrRVp3+VKlvCrIwu1aSs2UJwb9+QbAjD2OC\n+crZlw1Bk8gpMlwd/KPVUSpVJ/ii76Q+04RSH2QdpnjHMnI2LyE8dQMBpoRCE85yxwB2NR9JRO/R\nXNKnA+PahLP9H8OtjlYpVU9pQqmLjIG07eRtXUph0lKaZ20nEDjhjGYRl5Pedgyd40dwSc9objrj\nxcXe0U2tiVkpVe9pQqkrHCWYA//l+KbFBOz+jKYFh2kC7HR2ZWHgZAq7Xk58v4FM7tKCkEB75efR\nGeiUUj6iCcWfFeVRtHM1GZsSCT/wBU0cWYSaANY7+7Cj6bUE976SIXG9uDc6XOczUUpZThOKv8lN\nJ3vrUnI2LyUq/VuCTBGNTWPWmP4caX0JUf3GM7xPJ0aF6RhcSvna/ltvA6DD/71jcSR1gyYUP2CO\n7eHoD4swO5bROnsr4RiyTSSLA8aQ23EsnRNGc1nX1uduylJK+b3Q0FByc3PZvHkz9957L9nZ2djt\ndh555BFuvPHG08pOnTqV9evXU1RUxL59+7jgggsAePTRR7nhhhs8ul5iYiJ79uxh+vTpXr+XimhC\nsYLTSeGBH0jZ8DGN935Oy8JfiQaSnR1YF3oz9LiCvv2HM7GNNmUpVR95c/j6kpISAgIq/lF+7bXX\nej/4c9CEUltKCslMXk36xkSijnxBM0cGMcbGRnrxReRUQuMmMKhfP24KC7Y6UqWUj3Xv3r1sufzw\n9eUTyrlcdNFFjBw5kq+//prrrruOTp068fTTT1NUVERUVBTvvvsuLVu2ZO7cuWUjDU+ePJkWLVrw\nww8/cPToUZ5//nmvJxxNKD5kTp7g0A9LyN/2CTHH1tOMfAJNMBvsF3K8wxiiEyaQ0LMzgwO0KUup\n2nT06acp3HH+4esLfnaVKe1LOZfgnj1o/fDDVY6lOsPXg2twyXXr1gFw4sQJJkyYgIgwe/Zsnn/+\neWbNmnXWMWlpaaxfv55t27YxadIkTSj+riB9P/u//QD77s/omLuZdjhIN035ptHFFHUdR7fB47mk\nbZQ2ZSmlqjV8fambbrqpbPnAgQNMmjSJo0ePUlhYeFoNqLxrrrkGEaFv374cPny4RrFXRBNKTRnD\nsV82ceS7D2l6YBUdivZwAfCLacOaiIkE9plA30GjuCyskdWRKqXcPK1J+PIpr+oOX1+qSZMmZctT\np07l4YcfZvz48axevZqZM2dWeExw8KkmdWNM1YM+D00o1eAsKWbfplVk/bSEtqlf0MqZRnMjJNkv\nYFWb/yGi/zXExiXQRZuylFIVqMnw9RXJysqibdu2GGOYP3++FyKsHk0oHsrPzWLXt0twbF9Gl8z1\ndCGXQhPItuB+bO84hfZDrie2Yyf6alOWUuo8Soevz8jI4O233wYoG8q+Ov72t79x7bXXEhMTw8CB\nA0lJSfFitJ4TX1R7/FVCQoKpzjDOG169jfhjywmWYjJNKD+HD8VcMJ4Lhl1N84jmPohUKeVtO3bs\noGfPnlU6piG+2FjRn5OI/GiMOe8kSlpD8YAzvB0/2a+jSd8JXDDwMgYHBVkdklKqFjSkROINmlA8\nMOT2p6wOQSml/J4lUwCLSHMRWSUiu93fERWUuURENpf7FIjINe59nURkg/v490VEqwxKKWUxq+aU\nnwGsMcZ0A9a4109jjPnSGBNvjIkHRgEngZXu3bOAF93HnwDuqp2wlVJ1WUPqM66Omv75WJVQrgZK\nn22bD1xznvI3AJ8ZY06K643AUcBHVTheKdXAhYSEkJGRoUmlEsYYMjIyCAmp/kjmVvWhtDLGpAAY\nY1JEpOV5yt8EvOBebgFkGmNK3OuHgLaVHSgiU4ApAO3bt69R0EqpuismJoZDhw6Rnp5udSh+KyQk\nhJiYmGof77OEIiKrgdYV7HqkiueJBmKBz0s3VVCs0l85jDGvA6+D67HhqlxbKVV/BAYG0qlTJ6vD\nqNd8llCMMaMr2yciqSIS7a6dRANp5zjVJCDRGFPsXj8GNBORAHctJQY44rXAlVJKVYtVfShLgdvd\ny7cDS85R9mbgvdIV42oA/RJXv4onxyullKoFViWUmcAYEdkNjHGvIyIJIjK3tJCIdATaAV+dcfyD\nwJ9EZA+uPpU3ayFmpZRS59Cghl4RkXRgfzUPj8TV3FYf1Jd7qS/3AXov/qq+3EtN76ODMSbqfIUa\nVEKpCRHZ6MlYNnVBfbmX+nIfoPfir+rLvdTWfVjV5KWUUqqe0YSilFLKKzSheO51qwPwovpyL/Xl\nPkDvxV/Vl3uplfvQPhSllFJeoTUUpZRSXqEJpQpE5EkR2eoeTn+liLSxOqbqEpFnReRn9/0kikgz\nq2OqDhGZKCLJIuIUkTr5NI6IjBORnSKyR0TOGnm7rhCRt0QkTUSSrI6lJkSknYh8KSI73P9t/cHq\nmKpLREJE5HsR2eK+l8d9ej1t8vKciIQbY7Ldy78Hehlj7rE4rGoRkcuAL4wxJSIyC8AY86DFYVWZ\niPQEnMAcYJoxpupzPFtIROzALlwv+B4CfgBuNsZstzSwahCREUAu8I4xpo/V8VSXezioaGPMJhEJ\nA34ErqmjfycCNDHG5IpIIPAN8AdjzHe+uJ7WUKqgNJm4NeEcg1L6O2PMynIjNn+Ha0y0OscYs8MY\ns9PqOGpgILDHGLPXGFMELMQ1vUOdY4xZBxy3Oo6aMsakGGM2uZdzgB2cY0Rzf2Zcct2rge6Pz35u\naUKpIhF5SkQOArcAj1kdj5f8BvjM6iAaqLbAwXLr55yOQdUu9/BP/YAN1kZSfSJiF5HNuAbhXWWM\n8dm9aEI5g4isFpGkCj5XAxhjHjHGtAMWAPdZG+25ne9e3GUeAUpw3Y9f8uQ+6rAqTcegao+IhAIf\nA/ef0TpRpxhjHO6Zb2OAgSLis+ZIqybY8lvnGnb/DP8BPgX+6sNwauR89yIitwNXApcaP+5Mq8Lf\nSV10CNcAqKV0OgY/4O5v+BhYYIxZZHU83mCMyRSRtcA4wCcPTmgNpQpEpFu51QnAz1bFUlMiMg7X\nqM0TjDEnrY6nAfsB6CYinUQkCNfspEstjqlBc3dkvwnsMMa8cL7y/kxEokqf4BSRRsBofPhzS5/y\nqgIR+Ri4ANdTRfuBe4wxh62NqnrcQ/8HAxnuTd/VxSfWRORa4FUgCsgENhtjxlobVdWIyHjgJcAO\nvGWMecrikKpFRN4DLsY1sm0q8FdjTJ2bWkJELgK+Brbh+rcO8LAxZrl1UVWPiPQF5uP6b8sGfGCM\necJn19OEopRSyhu0yUsppZRXaEJRSinlFZpQlFJKeYUmFKWUUl6hCUUppZRXaEJRyotEJPf8pc55\n/Eci0tm9HCoic0TkF/dIsetEZJCIBLmX9cVk5Vc0oSjlJ0SkN2A3xux1b5qLa7DFbsaY3sAdQKR7\nEMk1wI2WBKpUJTShKOUD4vKse8yxbSJyo3u7TUT+5a5xLBOR5SJyg/uwW4Al7nJdgEHAo8YYJ4B7\nROJP3WUXu8sr5Te0yqyUb1wHxANxuN4c/0FE1gHDgI5ALNAS19Dob7mPGQa8517ujeutf0cl508C\nBvgkcqWqSWsoSvnGRcB77pFeU4GvcCWAi4APjTFOY8xR4Mtyx0QD6Z6c3J1oitwTQCnlFzShKOUb\nFQ1Lf67tAPlAiHs5GYgTkXP9Gw0GCqoRm1I+oQlFKd9YB9zontwoChgBfI9rCtbr3X0prXANplhq\nB9AVwBjzC7AReNw9+i0i0q10DhgRaQGkG2OKa+uGlDofTShK+UYisBXYAnwBPOBu4voY1xwoScAc\nXDMBZrmP+ZTTE8zdQGtgj4hsA97g1FwplwB1bvRbVb/paMNK1TIRCTXG5LprGd8Dw4wxR93zVXzp\nXq+sM770HIuAh4wxO2shZKU8ok95KVX7lrknPQoCnnTXXDDG5IvIX3HNKX+gsoPdE3Et1mSi/I3W\nUJRSSnmF9qEopZTyCk0oSimlvEITilJKKa/QhKKUUsorNKEopZTyCk0oSimlvOL/A1d5eaC6+b8H\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x147596d8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    plt.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'neg-logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的正确率（score）。可以看出在训练集上C越大（正则越少）的模型性能越好；但在测试集上当C=100时性能最好（L1正则和L2正则均是）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用LogisticRegressionCV实现正则化的 Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 4.0, 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5.0, 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7, 5.8, 5.9, 6.0, 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8, 6.9, 7.0, 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 7.7, 7.8, 7.9, 8.0, 8.1, 8.2, 8.3, 8.4, 8.5, 8.6, 8.7, 8.8, 8.9, 9.0, 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7, 9.8, 9.9],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=1, penalty='l1', random_state=None, refit=True,\n",
       "           scoring='neg_log_loss', solver='liblinear', tol=0.0001,\n",
       "           verbose=0)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "#Cs = [1e-06, 1e-05, 0.0001, 0.001, 0.01, 0.1,1,10,100,1000]\n",
    "Cs=[i/10 for i in range(1,100)]\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l1', solver='liblinear')\n",
    "lrcv_L1.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[-0.51625396, -0.52646867, -0.53270669, -0.53644476, -0.53873574,\n",
       "         -0.54029789, -0.54145358, -0.54235322, -0.54307242, -0.5436578 ,\n",
       "         -0.54413776, -0.54460259, -0.54499888, -0.54534916, -0.54565312,\n",
       "         -0.54591682, -0.54615688, -0.5463634 , -0.5465542 , -0.54672729,\n",
       "         -0.54687968, -0.54701573, -0.54714846, -0.54726912, -0.54738774,\n",
       "         -0.54748737, -0.54757121, -0.54766765, -0.5477511 , -0.54782393,\n",
       "         -0.54789099, -0.54796587, -0.54803026, -0.54808736, -0.54814855,\n",
       "         -0.5481986 , -0.54824926, -0.5482913 , -0.54833977, -0.54838822,\n",
       "         -0.54842953, -0.5484693 , -0.54850244, -0.54854005, -0.54857054,\n",
       "         -0.548597  , -0.54863305, -0.54866959, -0.54869864, -0.54872408,\n",
       "         -0.54876054, -0.54877099, -0.54880573, -0.54882755, -0.54884599,\n",
       "         -0.54887208, -0.54889364, -0.54891334, -0.54893575, -0.54895386,\n",
       "         -0.54896095, -0.5489937 , -0.54901399, -0.54903102, -0.54903859,\n",
       "         -0.54906206, -0.54907455, -0.54908667, -0.54909656, -0.54911538,\n",
       "         -0.54912417, -0.5491466 , -0.54915499, -0.54917527, -0.54918559,\n",
       "         -0.54919401, -0.54920626, -0.54920919, -0.54923394, -0.54923819,\n",
       "         -0.54924676, -0.54925844, -0.54926591, -0.54927024, -0.5492879 ,\n",
       "         -0.54929626, -0.54930413, -0.54931851, -0.54932106, -0.54932701,\n",
       "         -0.54934696, -0.54934965, -0.54935445, -0.5493657 , -0.54936335,\n",
       "         -0.54937943, -0.54938118, -0.54939417, -0.54940139],\n",
       "        [-0.49258757, -0.48294691, -0.47703938, -0.47412406, -0.47253248,\n",
       "         -0.4715302 , -0.47085283, -0.47036363, -0.46998955, -0.46969927,\n",
       "         -0.46947178, -0.46928516, -0.46911976, -0.46899039, -0.46887542,\n",
       "         -0.4687805 , -0.4686915 , -0.46861168, -0.46854159, -0.46848078,\n",
       "         -0.46842015, -0.46838318, -0.46833012, -0.46828953, -0.46825691,\n",
       "         -0.46821578, -0.46818621, -0.46815779, -0.46812858, -0.46810138,\n",
       "         -0.46808076, -0.46805825, -0.46803643, -0.46801751, -0.46799964,\n",
       "         -0.46798353, -0.4679673 , -0.46795067, -0.46793404, -0.46792287,\n",
       "         -0.46790796, -0.46789515, -0.467883  , -0.46787598, -0.46786044,\n",
       "         -0.46784751, -0.46783644, -0.46783279, -0.46781928, -0.46781206,\n",
       "         -0.46780625, -0.46779838, -0.46779001, -0.46778009, -0.46777781,\n",
       "         -0.46776649, -0.46776003, -0.46775174, -0.46774843, -0.4677463 ,\n",
       "         -0.46773611, -0.46772967, -0.46772877, -0.4677188 , -0.46771492,\n",
       "         -0.46770589, -0.46770409, -0.46769705, -0.46769482, -0.46769658,\n",
       "         -0.46768402, -0.46768523, -0.46769131, -0.46767576, -0.46767198,\n",
       "         -0.46766587, -0.46766349, -0.46766055, -0.46765696, -0.46765332,\n",
       "         -0.46765333, -0.46764424, -0.46764661, -0.46764279, -0.46763989,\n",
       "         -0.46763707, -0.46763213, -0.4676301 , -0.46762766, -0.46762869,\n",
       "         -0.46762087, -0.46762174, -0.46762096, -0.46761651, -0.46761142,\n",
       "         -0.46761776, -0.46760706, -0.46760695, -0.46760557],\n",
       "        [-0.48459815, -0.47665631, -0.47498648, -0.47436952, -0.47407393,\n",
       "         -0.4739094 , -0.47381088, -0.47374431, -0.47358471, -0.47340173,\n",
       "         -0.47325526, -0.47313724, -0.47303903, -0.47295572, -0.47288234,\n",
       "         -0.47282075, -0.47276809, -0.47272092, -0.4726801 , -0.47264295,\n",
       "         -0.47260603, -0.47257641, -0.47254813, -0.47252318, -0.47249849,\n",
       "         -0.47247777, -0.47245758, -0.472444  , -0.47242415, -0.47240695,\n",
       "         -0.47239252, -0.47237836, -0.47236811, -0.47235437, -0.47234241,\n",
       "         -0.47233452, -0.47232178, -0.47231318, -0.47230427, -0.47229531,\n",
       "         -0.47228959, -0.47228021, -0.4722732 , -0.47226548, -0.47225989,\n",
       "         -0.47225053, -0.47224884, -0.47224121, -0.47223669, -0.47223414,\n",
       "         -0.47222815, -0.47222006, -0.47221533, -0.47221116, -0.47220644,\n",
       "         -0.47220554, -0.47219757, -0.47219442, -0.47218944, -0.47218562,\n",
       "         -0.47218488, -0.47218031, -0.47217596, -0.47217474, -0.47217027,\n",
       "         -0.47216764, -0.47216395, -0.47216316, -0.47216193, -0.47215823,\n",
       "         -0.47215625, -0.47215324, -0.47215195, -0.47214434, -0.47214406,\n",
       "         -0.47214246, -0.47214191, -0.47213674, -0.47213687, -0.47213486,\n",
       "         -0.47212909, -0.47213242, -0.47213121, -0.47212465, -0.47212363,\n",
       "         -0.4721222 , -0.47212189, -0.47211656, -0.47211669, -0.47211612,\n",
       "         -0.47211502, -0.4721135 , -0.47211415, -0.47211335, -0.47210829,\n",
       "         -0.47210905, -0.47210387, -0.47210494, -0.4721033 ],\n",
       "        [-0.48888232, -0.48613786, -0.48639047, -0.4867797 , -0.48687964,\n",
       "         -0.48685461, -0.48686761, -0.48689026, -0.48691956, -0.48694153,\n",
       "         -0.48696773, -0.48699296, -0.4870177 , -0.48703288, -0.48705422,\n",
       "         -0.48707045, -0.48708543, -0.48709847, -0.48711308, -0.48712473,\n",
       "         -0.48713585, -0.48714313, -0.48715638, -0.4871651 , -0.48717507,\n",
       "         -0.48718251, -0.48719057, -0.48719779, -0.48720343, -0.48721009,\n",
       "         -0.48721861, -0.48722231, -0.48722655, -0.48723056, -0.48723524,\n",
       "         -0.48724174, -0.48724579, -0.48724843, -0.48725543, -0.48725601,\n",
       "         -0.4872589 , -0.48726422, -0.48726945, -0.48727182, -0.48727484,\n",
       "         -0.48727391, -0.48727898, -0.48728142, -0.48728316, -0.48728522,\n",
       "         -0.48728832, -0.48729106, -0.4872889 , -0.48729334, -0.48729699,\n",
       "         -0.48729931, -0.48730136, -0.48730354, -0.48730371, -0.48730785,\n",
       "         -0.48730885, -0.48731045, -0.48731136, -0.48731353, -0.48731443,\n",
       "         -0.48731038, -0.48731837, -0.48731753, -0.48731892, -0.48732094,\n",
       "         -0.48732189, -0.48732129, -0.48732585, -0.48732885, -0.48732851,\n",
       "         -0.48733108, -0.48732958, -0.48732864, -0.48733093, -0.48733363,\n",
       "         -0.48733388, -0.48733489, -0.48733538, -0.48733564, -0.48733798,\n",
       "         -0.48733817, -0.48733912, -0.48734149, -0.48734098, -0.48734266,\n",
       "         -0.48734309, -0.48734109, -0.48734254, -0.48734123, -0.48734534,\n",
       "         -0.48734592, -0.48734526, -0.48734944, -0.48734698],\n",
       "        [-0.44347412, -0.42973959, -0.42593232, -0.42421791, -0.42320425,\n",
       "         -0.42255774, -0.4221212 , -0.42179602, -0.42155378, -0.42135117,\n",
       "         -0.42120553, -0.42107935, -0.42097369, -0.42088002, -0.42080048,\n",
       "         -0.42073362, -0.42067313, -0.42062231, -0.42057596, -0.42053187,\n",
       "         -0.42049696, -0.42046338, -0.42043114, -0.42040328, -0.42037311,\n",
       "         -0.42035299, -0.42033166, -0.42031252, -0.4202931 , -0.42027695,\n",
       "         -0.42025859, -0.42023586, -0.42022535, -0.4202143 , -0.42020125,\n",
       "         -0.42018894, -0.42017757, -0.42016493, -0.42015598, -0.42014485,\n",
       "         -0.420137  , -0.4201259 , -0.42011859, -0.42011591, -0.4201013 ,\n",
       "         -0.42009144, -0.42009103, -0.42008516, -0.42007807, -0.42007233,\n",
       "         -0.42006896, -0.42005886, -0.42005124, -0.42004842, -0.42004063,\n",
       "         -0.42004136, -0.42003776, -0.42003247, -0.42002621, -0.42002228,\n",
       "         -0.42001618, -0.42001334, -0.42000921, -0.42000904, -0.42000261,\n",
       "         -0.42000077, -0.41999636, -0.4199906 , -0.41999063, -0.41998916,\n",
       "         -0.41998277, -0.41997996, -0.41998196, -0.41997765, -0.41997509,\n",
       "         -0.41997017, -0.41996602, -0.41996624, -0.41996578, -0.41996371,\n",
       "         -0.41995526, -0.41995577, -0.41995588, -0.41995416, -0.41994937,\n",
       "         -0.41994447, -0.4199455 , -0.41994442, -0.41994214, -0.41993849,\n",
       "         -0.41993683, -0.41993637, -0.4199347 , -0.41993297, -0.41993191,\n",
       "         -0.4199331 , -0.41992991, -0.41992425, -0.41992521]])}"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.48515922 -0.48038987 -0.47941107 -0.47918719 -0.47908521 -0.47902997\n",
      " -0.47902122 -0.47902949 -0.479024   -0.4790103  -0.47900761 -0.47901946\n",
      " -0.47902981 -0.47904163 -0.47905312 -0.47906443 -0.479075   -0.47908336\n",
      " -0.47909299 -0.47910152 -0.47910773 -0.47911637 -0.47912285 -0.47913004\n",
      " -0.47913826 -0.47914328 -0.47914745 -0.47915595 -0.47916007 -0.47916386\n",
      " -0.47916829 -0.47917213 -0.47917734 -0.47918082 -0.47918542 -0.47918947\n",
      " -0.47919234 -0.4791937  -0.4791979  -0.47920145 -0.4792046  -0.47920696\n",
      " -0.47920934 -0.47921385 -0.4792134  -0.47921208 -0.47921767 -0.47922204\n",
      " -0.47922317 -0.47922557 -0.47923045 -0.47922787 -0.47923024 -0.47923211\n",
      " -0.47923357 -0.47923696 -0.47923807 -0.4792391  -0.47924071 -0.47924318\n",
      " -0.47924139 -0.47924549 -0.47924786 -0.47924943 -0.47924816 -0.47924935\n",
      " -0.47925146 -0.479251   -0.47925257 -0.47925606 -0.47925382 -0.47925727\n",
      " -0.47926121 -0.47926037 -0.47926104 -0.47926072 -0.47926145 -0.47926027\n",
      " -0.47926489 -0.47926474 -0.47926366 -0.47926515 -0.479267   -0.4792655\n",
      " -0.47926775 -0.47926764 -0.47926855 -0.47927022 -0.47926971 -0.47927059\n",
      " -0.47927256 -0.47927247 -0.47927336 -0.47927395 -0.47927206 -0.47927705\n",
      " -0.47927346 -0.47927595 -0.47927649]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAEKCAYAAAArYJMgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8XPV57/HPI40WW7Jsy7Jls9hm\n8YJJwIBIQoAEY5OkSYtJkxAopCYh5QZC7+1NSWtKeltCm9CmJKUhLSFAMW02AqGQhCRgGTAhIUEG\nYwMj44WAVy3etFjrzHP/OEf2ILSM5DmakfR9v17zmjNnm2eORnr0+53feY65OyIiIlHJy3YAIiIy\ntinRiIhIpJRoREQkUko0IiISKSUaERGJlBKNiIhESolGREQipUQjIiKRUqIREZFIxbIdQC6oqKjw\nuXPnZjsMEZFRZd26dY3uPn2w9ZRogLlz51JTU5PtMERERhUzeyOd9dR1JiIikVKiERGRSCnRiIhI\npJRoREQkUllJNGZWbmZPmNnm8HnqAOuWmdlOM7sjfD3JzNanPBrN7F/DZXPMrNrMNpjZU2Z23Eh9\nJhER6Vu2WjQrgWp3nwdUh6/7cwvwdM8Ld29298U9D+AN4Mfh4n8B7nf304AvA1+NJHoREUlbthLN\ncmBVOL0KuKSvlczsLKASeLyf5fOAGcAz4axFBIkL4MnwfUREJIuydR1NpbvvBnD33WY2o/cKZpYH\n3AZ8Cljaz34uB37oR+5H/RLwMeB24KPAJDOb5u57M/0BZHxzdzoTSTq7kyQdimJ5FObnkZdnw9pf\nR3eCprZumtq7aGrroqm9O3zuorM7SSLpdCedRMojlmeUTShgcvh463SMolh+hj+1yPBElmjMbDUw\ns49FN6W5i+uAx9x9u1m/v7yXESSiHjcAd5jZVcBaYCfQ3U981wDXAMyePTvNkGQs6k4k2bG/jdcb\nW9nW2MrvG1t5vbGVfa2dh5NJZ3fybdN9KczPC5JOLHguKsinMD8PM+hKJOlKeK/n5OHXmVYUy6Ns\nQgFlxTEmTyigvKSQqRMLKS8pZMrEQspLCigvKUp5LqSsOMYAv28iw2JHGgMj+KZmm4ALwtbMLOAp\nd1/Qa53vAucDSaAUKAT+3d1XhstPB37k7vP7eY9SoNbdBx0QUFVV5aoMMLYlk05dczuvNwTJ5PWU\nhPLmvkN0J4/8HkwqjnFiRQkVpUWHE0ZhzyM///B0TyvGjMNJqKM7SUdXko7uxJHX3QmSTrh9HrE8\noyCcLsg3Yvl5lBbFKCuOhYmhp3USY1JxAcWxfPLyIJaXd+TZoCvhNLV3cbAtaAUdDB+HW0Nhi6ip\nrZv9hzrZf6iLA4c62dvaSWd334myIN8oLylkWkkR00oLmVZSyLTSI9NTJwZJaurEAiZPLGDqxEIK\n8jV4dbwys3XuXjXYetnqOnsUWAHcGj4/0nsFd7+iZzpsoVT1JJnQ5cD3U7cxswpgn7sngRuBezMe\nuYwKBw918eL2/bzw5gFefHM/67cfoLn9SOO2uCCPudNKWDBzEh96x0xOqCg5/CgvKRwV/9UXxoyK\n0iIqSouGtJ2709aVYG9LJ/vDxLO/tZN9rcH03pYO9rZ00tjaye/3trK3pZNDnYl+9zdlYkGQkMJW\nUXmYlCYfTpoxpkwspKK0iOmlRZRNUKtpvMlWorkVeMDMrgbeBD4BYGZVwOfc/bNp7ONS4MO95l0A\nfNXMnKDr7PMZi1hyViLpbKlv4YU39/PCG/t54c39bG1oBSDPYH7lJP7o9GM4ZVYZJ1WUMLeihJll\nxcM+nzLamRkTC2NMLI9xfPnEtLZp60zQ2NLBgUNdHGjrDJ4P9SSmTva2dtDY0snWhhae/32QwJL9\ndJbE8owpE4NzST1dedNKg0Q0Y1IR0ycVHz7PVFZcQGlRjJKiGIUxtZxGq6x0neUadZ2NLgcOdfLi\n9gO8+EbQYnlp+wGaO4LWytSJBZwxeypnzp7CmbOnctrxUygtUu3YkZZMOi2dPV14QdddY0sHDc0d\nb+nG29/aFbakOtjX2n9yguBnO2vyBComFTGpOOhqLC2KUVoUJKWeFlRpcexwAps8oUAJKkK53nUm\nMiTx3U08uG4HT26qZ1tKa2XhzDIuXnwMZ86eyplzpjJ32kR1y+SAvDwLus2KC6Dfy7HfqjuRZF9r\nJ/XNHYfPLTW3d9Ha0U1Tezd7mtrZfaCNfa2d7Nh/iOb2blo7ugfs1oNgUMSU8HxST0uqtKiA0qLg\nfFtxQT5TJxYyteTIqL3SogJKivIpKVRrKhOUaCRn7W/t5JH1O/nRuh28squJgnzjvJMr+NiZx3HG\n7CmcftwUStRaGTNi+XnMKCtmRlnxkLbrTiRpbu/mQFsXze1dNLcHLah9rZ0cPNRFc0c3Bw91sbe1\nk6a2Ll5vbKWlvZvWzgRdiSRtXQkG69iZPKGAiYX5dCedzu4kZRNiHDtlApVlxUwsjDGhIJ/JEwqY\nObmIRBIcD1tbMSYUBolsUnHscGKbWJBPbBwNotBvqeSU7kSStZsb+FHNDlbH6+hKOO84toy//6NF\nLF98LFNLCrMdouSYWH4eU0sKh/3dSCSdprYuDrR1sf9QJ83hqL1DnUFrqbm9m4bmDjq6E+SZURjL\n42BbFzv3t/Himwdo60rQ1pmgpaPPKyn6NaEgn65EkorSYBBFaVGMhDv5ecakohj5eUZRQT4VpYUk\nks7UiYUcO2UCGOSZYYAD5SUFTCiI0Z1MMqm4gDyD4oJ8YnlGabifCYX5TCjIx8xw9xFv9SvRSE7Y\nUt/Mj2p28OMXd9LQ3EF5SSGfes9cPlF1HKfMKst2eDKG5efZ4UR1AiXD3k9nd5K6pvbDw72b27s4\n1JngUGeC/Yc6aenopqMrQXtXkpaOoNuvIJZHY3Nwfqqlo5ui/Dy6k87ug+0kkk57d4LG5g7y84zm\nju5BW16DKczPw3FKimIkk86k4gK++MEFXHLGsUe340Eo0UjWHGzr4icv7eLBdTtYv/0A+XnGkgUz\n+ETVcSxZMEP94jKqFMby3jKKb+bkoXUBDqajO0F9UwcA7pB0xwwaWzro6E4Sy8ujub2LpEN7V4Lu\nZJKWjgTJZDCc/VBHN50Jx3EOtHZRGMujtbObGZOGNjx+OJRoZEQlks6zWxp5cN0OfvnKHjq6kyyo\nnMSXPnIKyxcfy/QR+NKLjEZFsfw+h6PPmTb8VthIUaKREfF6YysPrdvBQy/sYPfBdiZPKOCTZx/P\nJ846nnccW6aRYiJjmBKNROrZLY386+rXeP73+8kzeP/86XzpI4tYtmiGij6KjBNKNBKZF9/cz2fu\ne54ZZUX89YcW8sdnHkvlEIeuisjop0Qjkdi+7xB/dn8NlWXFPHzde5k2xHpcIjJ2KNFIxjW1d3H1\nqufp7E7yg2vOVpIRGeeUaCSjuhJJPv/dF9jW0Mr9n3kXJ88ozXZIIpJlSjSSMe7O3z36Cs9sbuSf\nP3Ya7z25ItshiUgO0BVxkjH3/Op1vvfbN7n2gpO49Ozjsx2OiOQIJRrJiF++sod/fCzOh985ky9+\nYMHgG4jIuKFEI0dt446D/MUP1nPacVP4+qWLx+0NxUSkb0o0clR2HWjj6lXPU15SyN1/WkVxgS7C\nFJG3UqKRYWvp6OYz9z1PW2eCe686W3XKRKRPGnUmw9KdSPLn33uBzfUt3HvV2SyYOSnbIYlIjlKL\nRoblH34W58lNDdx88am8f/70bIcjIjlMiUaG7L5nX+e+X/+ez553Ale+Z062wxGRHKdEI0OypraO\nL//0VZadUsmNHz4l2+GIyCigRCNpe3VXE3/+vRdZdEwZ/3b5YvI1jFlE0pC1RGNm5Wb2hJltDp+n\nDrBumZntNLM7UuZdbmYbzWyDmf3CzCqGul9JX11TO1evep5JxQXcs+JsJhZqHImIpCebLZqVQLW7\nzwOqw9f9uQV4uueFmcWA24El7n4asAG4fhj7lTQc6uzm6lXPc7Cti3uuqtI9ZURkSLKZaJYDq8Lp\nVcAlfa1kZmcBlcDjqbPDR4kF9wAuA3YNZb+SnkTS+T8/WM+ru5q440/O4NRjJmc7JBEZZbKZaCrd\nfTdA+Dyj9wpmlgfcBnwxdb67dwHXAhsJEswi4J509yvpu/XncZ54tY6//cNFXLiwMtvhiMgoFGmi\nMbPVZvZyH4/lae7iOuAxd9/ea78FBInmDOAYgq6zG4cY2zVmVmNmNQ0NDUPZdNz47m/f4DvPvM6K\nc+bw6XNPyHY4IjJKRXpG192X9bfMzOrMbJa77zazWUB9H6udA5xvZtcBpUChmbUAD4X73xru6wGO\nnItJZ7+4+13AXQBVVVU+vE84dj39WgP/75FXWLJgOn/7h4uyHY6IjGLZ7Dp7FFgRTq8AHum9grtf\n4e6z3X0ucANwv7uvBHYCi8ys55L0i4B4uvuVgW3a08z1332BeTNK+eafnEksX6PgRWT4svkX5Fbg\nIjPbTJAobgUwsyozu3ugDd19F3AzsNbMNgCLga8MtF9JT0NzB5+573mKC/O596qzKS3SMGYROTrm\nrl6jqqoqr6mpyXYYWdfeleCTdz3Hpj1NPPC/zuG046ZkOyQRyWFmts7dqwZbT/+uCgDJpPOFB9az\nYccB7rzyLCUZEckYdb4LAF97fBOPbdzDjX+wkA+eOjPb4YjIGKJEIzzw/Hb+46mtXP6u2fzZ+Sdm\nOxwRGWOUaMa5X29p5G8e3sj58yr48vJTCQotiIhkjhLNOLalvoXP/fc6Tqgo4VtXnEmBhjGLSAT0\nl2Wc2tsSDGMujOVx71VnU1ZckO2QRGSM0qizcai9K8E1/7WOuqZ2vn/Nezi+fGK2QxKRMUyJZhz6\n9ye3sO6N/XzrT87kzNm6XY+IREtdZ+PQYy/v4b0nTeMjp83KdigiMg4o0Ywzb+xtZUt9C0tPUcl/\nERkZSjTjTHU8KGa97BTdpkdERoYSzThTXVvHSdNLmDOtJNuhiMg4oUQzjjS3d/HbbftYpm4zERlB\nSjTjyNrXGulOus7PiMiIUqIZR6rjdUyZWMCZs1WZWURGjhLNOJFIOk9uqueC+dN1x0wRGVH6izNO\nvPjmfvYf6lK3mYiMOCWacWJ1vJ5YnvG++dOzHYqIjDNKNONEdbyOs+eWM3mCimeKyMhSohkH3tx7\niM31LSzVRZoikgVKNONAdW0dgK6fEZGsUKIZB6rj9Zw4vYS5FaoGICIjT4lmjGtu7+K3r+9Va0ZE\nskaJZox7ZnMjXQln6UKdnxGR7MhKojGzcjN7wsw2h8/93n3LzMrMbKeZ3ZEy73Iz22hmG8zsF2ZW\nEc7/hJm9YmZJM6saic+S61bH65g8oYCz5ugGZyKSHdlq0awEqt19HlAdvu7PLcDTPS/MLAbcDixx\n99OADcD14eKXgT8G1kYR9GiTSDpPbWrgggWqBiAi2ZOtvz7LgVXh9Crgkr5WMrOzgErg8dTZ4aPE\nzAwoA3YBuHvc3TdFFfRos377fva1dqoagIhkVbYSTaW77wYIn992AsHM8oDbgC+mznf3LuBaYCNB\nglkE3DPUAMzsGjOrMbOahoaGoX+CUWB1vJ78POP9qgYgIlkUWaIxs9Vm9nIfj+Vp7uI64DF3395r\nvwUEieYM4BiCrrMbhxqfu9/l7lXuXjV9+tj8QxxUA5iqagAiklWxqHbs7sv6W2ZmdWY2y913m9ks\noL6P1c4Bzjez64BSoNDMWoCHwv1vDff1AAOf4xmXtu87xGt1LXzpI6dkOxQRGeey1XX2KLAinF4B\nPNJ7BXe/wt1nu/tc4AbgfndfCewEFplZTzPkIiAefcijS3U8qAag8zMikm3ZSjS3AheZ2WaCRHEr\ngJlVmdndA23o7ruAm4G1ZrYBWAx8Jdz+o2a2g6A19DMz+2WEnyGnVdfWc2JFCSeoGoCIZJm5e7Zj\nyLqqqiqvqanJdhgZ09LRzRlffpyr3juXmz6yKNvhiMgYZWbr3H3QaxZ1ccUY9MxrDUE1AHWbiUgO\nUKIZg1bH6ykrjqkagIjkBCWaMSaoBlDPBQtmUKBqACKSA9L6S2Rm55pZSTh9pZl93czmRBuaDMf6\n7QfY29qpm5yJSM5I91/e/wAOmdnpwF8BbwD3RxaVDFt1vI78POOC+Uo0IpIb0k003R4MT1sO3O7u\ntwOTogtLhmtNbT1Vc6YyeaKqAYhIbkg30TSb2Y3AlQTXp+QD+kuWY3bsP0Ttnmbd5ExEckq6ieaT\nQAdwtbvvAY4FvhZZVDIs1fGgko/Oz4hILkm31lkzQZdZwszmAwuB70cXlgxHdW09J1SUcOL00myH\nIiJyWLotmrVAkZkdS3Cjsk8D90UVlAxdS0c3z23dq1s2i0jOSTfRmLsfIrh75Tfd/aPAqdGFJUP1\nq80NdCaSqgYgIjkn7URjZucAVwA/C+flRxOSDMfqeD2TimNUzVU1ABHJLekmmr8guLnYw+7+ipmd\nCDwZXVgyFMmk82StqgGISG5KazCAuz8NPG1mk8ys1N23Af872tAkXet3BNUAlmm0mYjkoHRL0LzT\nzF4EXgZeNbN1ZqZzNDmipxrA++ePzVtSi8jolm4/y7eBL7j7HHefDfwl8J3owpKhqI7Xc9acqUyZ\nWJjtUERE3ibdRFPi7ofPybj7U4Bu3ZgDjlQDULeZiOSmdC/Y3GZmfwv8V/j6SuD1aEKSoVhTG1QD\nuHChhjWLSG5Kt0XzGWA68GPg4XD601EFJemrjtczd9pETpquBqaI5KZ0R53tR6PMck5rRze/2bqX\nT50zBzPLdjgiIn0aMNGY2U8A72+5u1+c8Ygkbc9sbgyrAej8jIjkrsFaNP8yIlHIsKyprWNScYyz\n55ZnOxQRkX4NmGjCCzUlByWTzpraBt4/f7qqAYhITkv3gs2NZrah1+MZM/uGmU0b6puaWbmZPWFm\nm8Pnfgt0mVmZme00sztS5l2eEtMvzKwinP81M6sN5z9sZlOGGtto8dKOAzS2dOgmZyKS89L9V/jn\nBMU0rwgfPwGeAfYwvNsFrASq3X0ewW0HVg6w7i3A4ZaVmcWA24El7n4asAG4Plz8BPCOcP5rBPXZ\nxqQ1tfXkGaoGICI5L93raM5193NTXm80s2fd/Vwzu3IY77scuCCcXgU8Bfx175XM7CygEvgFUNUz\nO3yUmNleoAzYAuDuj6ds/hzw8WHENiqsjtdTNaecqSWqBiAiuS3dFk2pmb2754WZvQvouY1j9zDe\nt9LddwOEz28bNmVmecBtwBdT57t7F3AtsBHYBSwC7unjPT5D0BLrk5ldY2Y1ZlbT0NAwjI+QPTsP\ntBHf3aTRZiIyKqTbovkscK+ZlRK0JpqAq82sBPhqXxuY2WpgZh+LbkrzPa8DHnP37anXiJhZAUGi\nOQPYBnyToIvsH1LWuYkgAX63v527+13AXQBVVVX9DuHORWvidQBKNCIyKqR7webzwDvNbDLB3TYP\npCx+oJ9tlvW3PzOrM7NZ7r7bzGYB9X2sdg5wvpldR9B6KjSzFuChcP9bw309QMo5HjNbAfwhsNTd\nR1UCSVd1bT1zpk3kpOmlg68sIpJl6Y46m2xmXyc4cb/azG4Lk85wPQqsCKdXAI/0XsHdr3D32e4+\nF7gBuN/dVwI7gUVm1nMW/CIgHsb5IYJzPReHt54ecw51dvPrrXtZurBS1QBEZFRI9xzNvUAzcGn4\naAL+8yje91bgIjPbTJAobgUwsyozu3ugDd19F3AzsNbMNgCLga+Ei+8AJgFPmNl6M7vzKGLMSc9s\nbqSzW9UARGT0SPcczUnu/rGU1zeb2frhvqm77wWW9jG/huB8UO/595EyjNrd7wTelkTc/eThxjRa\nrInXM6lI1QBEZPRIt0XTZmbn9bwws3OBtmhCkv4kk051bT3vWzCdwpiqAYjI6JBui+ZaYFXPYABg\nH3BVVEFJ3zbsPBhWA1C3mYiMHumOOlsPnG5mZeHrpkijkj6tideRZ3DBfCUaERk9BrtNwBf6mQ+A\nu389gpikH6vj9Zw1Z6qqAYjIqDJYR/+kQR4yQnYdaOPV3U0sVRFNERllBrtNwM0jFYgMbE1tcE3r\n0oXqNhOR0WXIQ5fM7IUoApGBVcfrmF0+kZNnqBqAiIwuwxkjq8vRR9ihzm6e3bqXpafMUDUAERl1\nhpNofpbxKGRAv+qpBrBQ52dEZPQZcqJx9y9FEYj0b01tUA3gXSeoGoCIjD5pXUdjZs1A70rIB4Ea\n4C/dfVumA5PA4WoA81UNQERGp3QrA3yd4CZj3yM4R3MZwb1mNhEU3LwgiuAENu48SENzBxdqtJmI\njFLp/ov8IXf/trs3u3tTeNOwD7v7D4GpEcY37lXX1pNnsESJRkRGqXQTTdLMLjWzvPBxacqyMXlz\nsVxRHa/jzNlTKVc1ABEZpdJNNFcAnyK4E2ZdOH2lmU0Aro8otnFv98E2XtnVxIUqoikio1i6RTW3\nAX/Uz+JfZS4cSdVTDWCZys6IyCiW7q2c55tZtZm9HL4+zcw0zDli1fF6ji+fwDxVAxCRUSzdrrPv\nADcCXQDuvoFg5JlEpK0zwbNbGlm6sFLVAERkVEs30Ux099/1mted6WDkiGe3NNLRnWSpzs+IyCiX\nbqJpNLOTCEeYmdnHgd2RRSVU19ZRWhTj3SdMy3YoIiJHJd0LNj8P3AUsNLOdwOsEI9EkAsmkUx2v\n533zK1QNQERGvXQTzU7gP4EngXKgCVgBfDmiuMa1V3Y1Ud/cwYUqoikiY0C6ieYR4ADwAkEpGonQ\n6ngdZrBkwfRshyIictTSTTTHufuHMvnGZlYO/BCYC/weuNTd9/ezbhkQBx529+vDeZcDf0Nw3mgX\ncKW7N5rZLcByIElwgelV7j6qkmN1bVANYFppUbZDERE5aumeAPi1mb0zw++9Eqh293lAdfi6P7cA\nT/e8MLMYcDuwxN1PAzZwpELB19z9NHdfDPwU+H8ZjjtSew628/LOJhXRFJExI91Ecx6wzsw2mdkG\nM9toZhuO8r2XA6vC6VXAJX2tZGZnAZXA46mzw0eJBReZlBF26bl7U8p6JYyyWmyqBiAiY026XWd/\nEMF7V7r7bgB3321mb/sX3szygNsIaqst7Znv7l1mdi2wEWgFNhOMjOvZ7h+BPyW4Z86SCGKPTHW8\njuOmTmB+paoBiMjYkFaLxt3f6Osx2HZmttrMXu7jsTzN+K4DHnP37b32WwBcC5wBHEPQdXZjSrw3\nufvxwHfpp+inmV1jZjVmVtPQ0JBmONFq60zwqy2NLF04Q9UARGTMSLdFMyzuvqy/ZWZWZ2azwtbM\nLIIT972dA5xvZtcBpUChmbUAD4X73xru6wH6PsfzPeBnwN/1EdtdBNcGUVVVlRPda7/e2lMNQN1m\nIjJ2ZPNqwEcJrsUhfH6k9wrufoW7z3b3ucANwP3uvpLgup5FZtYz/vciglFpmNm8lF1cDNRGE37m\nrY7XU1KYz7tPLM92KCIiGRNpi2YQtwIPmNnVwJvAJwDMrAr4nLt/tr8N3X2Xmd0MrDWzLuAN4Kqe\n/ZrZAoLhzW8An4vuI2SOu7Omto73zZ9OUSw/2+GIiGRM1hKNu+8l5QR/yvwa4G1Jxt3vA+5LeX0n\ncGcf630sk3GOlFd2NVHX1KFhzSIy5qiQVo44XA1AiUZExhglmhxRHa/njOOnUKFqACIyxijR5IC6\npnY27jyo0WYiMiYp0eSAnmoAusmZiIxFSjQ5oDpex7FTJrCgclK2QxERyTglmixr7wqrAZyiagAi\nMjYp0WTZr7c20t6lagAiMnYp0WTZ6ng9EwvzeY+qAYjIGKVEk0Xuzpp4PefPq1A1ABEZs5RosuiV\nXU3saWpXt5mIjGlKNFlUHa/HDJWdEZExTYkmi6pr61isagAiMsYp0WRJfVM7G3YcZKlaMyIyxinR\nZMmRagA6PyMiY5sSTZasjtdz7JQJLJypagAiMrYp0WRBe1eCZ7c0cuFCVQMQkbFPiSYLfrN1L21d\nCRXRFJFxQYkmC1bH68JqANOyHYqISOSUaEaYu7Omtp7zTq6guEDVAERk7FOiGWGv7m5i98F2lmm0\nmYiME0o0I6w6HgxrXqLrZ0RknFCiGWHV8TpOP34K0yepGoCIjA9KNCOovrmdl3YcZJlaMyIyjmQt\n0ZhZuZk9YWabw+epA6xbZmY7zeyOlHmXm9lGM9tgZr8ws4pe29xgZt57fjY9qWoAIjIOZbNFsxKo\ndvd5QHX4uj+3AE/3vDCzGHA7sMTdTwM2ANenLD8euAh4M4K4h211vJ5jJhdzyixVAxCR8SObiWY5\nsCqcXgVc0tdKZnYWUAk8njo7fJRYcGl9GbArZfk3gL8CPMMxD1t7V4JfbW7kwlNUDUBExpdsJppK\nd98NED6/7cSFmeUBtwFfTJ3v7l3AtcBGggSzCLgn3OZiYKe7vxRp9EP0m2091QDUbSYi40ssyp2b\n2WpgZh+LbkpzF9cBj7n79tRWgJkVECSaM4BtwDeBG83s6+G+P5BGbNcA1wDMnj07zXCGrzpex4SC\nfM5RNQARGWciTTTuvqy/ZWZWZ2az3H23mc0C6vtY7RzgfDO7DigFCs2sBXgo3P/WcF8PEJzjeQQ4\nAXgpTEzHAS+Y2bvcfU+v2O4C7gKoqqqKtIvN3VkTr+e8eaoGICLjTza7zh4FVoTTKwiSxFu4+xXu\nPtvd5wI3APe7+0pgJ7DIzKaHq14ExN19o7vPcPe54TY7gDN7J5mRFt/dzK6D7SxTEU0RGYeymWhu\nBS4ys80EieJWADOrMrO7B9rQ3XcBNwNrzWwDsBj4SsTxDlt1vA5QNQARGZ/MPWcGZmVNVVWV19TU\nRLb/S771LO7OI9efF9l7iIiMNDNb5+5Vg62nygARa2ju4KUdBzTaTETGLSWaiD1ZW487usmZiIxb\nSjQRq66tY9bkYhbNKst2KCIiWaFEE6H2rgTPbG7kwoWqBiAi45cSTYSe27aXQ50J3eRMRMY1JZoI\nVcfrg2oAJ6kagIiMX0o0EXF31tTWc+7JqgYgIuObEk1Eavc0s/NAm6oBiMi4p0QTkZ5qABeqGoCI\njHNKNBGprq3ntOMmM6OsONuhiIhklRJNBBpbOli//QBLF2q0mYiIEk0E1qgagIjIYUo0EVgTr2dm\nWTGnHqNqACIiSjQZ1tGd4JlJJ17MAAALH0lEQVTNDVx4iqoBiIiAEk3GPbdtH62dCQ1rFhEJKdFk\n2Jp4HcUFebz3pIpshyIikhOUaDLI3Vkdr+c8VQMQETlMiSaDNtUF1QB0kzMRkSOUaDKoOl4PqBqA\niEgqJZoMqo7X8c5jJ1OpagAiIocp0WRIY0sHL24/oIs0RUR6UaLJkCfDagC6yZmIyFsp0WTImtp6\nKsuKVA1ARKSXrCQaMys3syfMbHP4PHWAdcvMbKeZ3ZEy73Iz22hmG8zsF2ZWEc7/+3Dd9eHjwyPx\neTq6E6x9rYELF1aqGoCISC/ZatGsBKrdfR5QHb7uzy3A0z0vzCwG3A4scffTgA3A9Snrf8PdF4eP\nxzIf+tv9VtUARET6la1EsxxYFU6vAi7payUzOwuoBB5PnR0+SixoPpQBu6ILdXBrauspiqkagIhI\nX7KVaCrdfTdA+Py2poCZ5QG3AV9Mne/uXcC1wEaCBLMIuCdllevDLrV7B+qSy5SgGkAd551cwYRC\nVQMQEektskRjZqvN7OU+HsvT3MV1wGPuvr3XfgsIEs0ZwDEEXWc3hov/AzgJWAzsJkhU/cV3jZnV\nmFlNQ0PD0D5citfqWtixX9UARET6E4tqx+6+rL9lZlZnZrPcfbeZzQLq+1jtHOB8M7sOKAUKzawF\neCjc/9ZwXw8QnuNx97qU9/gO8NMB4rsLuAugqqrKh/jxDquuDd5S1QBERPqWra6zR4EV4fQK4JHe\nK7j7Fe4+293nAjcA97v7SmAnsMjMpoerXgTEAcKk1eOjwMvRhH9EdbyedxxbxszJqgYgItKXbCWa\nW4GLzGwzQaK4FcDMqszs7oE2dPddwM3AWjPbQNBN9pVw8T/3DHsGlgD/N6oPALC3pYMX3tzP0oXq\nNhMR6U9kXWcDcfe9wNI+5tcAn+1j/n3AfSmv7wTu7GO9T2UyzsE8talB1QBERAahygBHoWxCARct\nqlQ1ABGRAWSlRTNWXLSokosWqTUjIjIQtWhERCRSSjQiIhIpJRoREYmUEo2IiERKiUZERCKlRCMi\nIpFSohERkUgp0YiISKTMfdiFi8cMM2sA3hjm5hVAYwbDyZRcjQtyNzbFNTSKa2jGYlxz3H36YCsp\n0RwlM6tx96psx9FbrsYFuRub4hoaxTU04zkudZ2JiEiklGhERCRSSjRH765sB9CPXI0Lcjc2xTU0\nimtoxm1cOkcjIiKRUotGREQipUSTBjP7hJm9YmZJM+t3dIaZfcjMNpnZFjNbmTL/BDP7rZltNrMf\nmllhhuIqN7Mnwv0+YWZT+1hniZmtT3m0m9kl4bL7zOz1lGWLRyqucL1Eyns/mjI/m8drsZn9Jvx5\nbzCzT6Ysy+jx6u/7krK8KPz8W8LjMTdl2Y3h/E1m9sGjiWMYcX3BzF4Nj0+1mc1JWdbnz3QEY7vK\nzBpSYvhsyrIV4c9+s5mtGOG4vpES02tmdiBlWSTHzMzuNbN6M3u5n+VmZv8WxrzBzM5MWZbZY+Xu\negzyAE4BFgBPAVX9rJMPbAVOBAqBl4BF4bIHgMvC6TuBazMU1z8DK8PplcA/DbJ+ObAPmBi+vg/4\neATHK624gJZ+5mfteAHzgXnh9DHAbmBKpo/XQN+XlHWuA+4Mpy8DfhhOLwrXLwJOCPeTP4JxLUn5\nDl3bE9dAP9MRjO0q4I4+ti0HtoXPU8PpqSMVV6/1/xy4N+pjBrwPOBN4uZ/lHwZ+DhjwHuC3UR0r\ntWjS4O5xd980yGrvAra4+zZ37wR+ACw3MwMuBB4M11sFXJKh0JaH+0t3vx8Hfu7uhzL0/v0ZalyH\nZft4uftr7r45nN4F1AODXpA2DH1+XwaI90FgaXh8lgM/cPcOd38d2BLub0TicvcnU75DzwHHZei9\njzq2AXwQeMLd97n7fuAJ4ENZiuty4PsZeu9+uftagn8s+7McuN8DzwFTzGwWERwrJZrMORbYnvJ6\nRzhvGnDA3bt7zc+ESnffDRA+zxhk/ct4+xf8H8Nm8zfMrGiE4yo2sxoze66nO48cOl5m9i6C/1C3\npszO1PHq7/vS5zrh8ThIcHzS2TbKuFJdTfBfcY++fqaZkm5sHwt/Rg+a2fFD3DbKuAi7GU8A1qTM\njvKYDaS/uDN+rGJHs/FYYmargZl9LLrJ3R9JZxd9zPMB5h91XOnuI9zPLOCdwC9TZt8I7CH4Y3oX\n8NfAl0cwrtnuvsvMTgTWmNlGoKmP9bJ1vP4LWOHuyXD2sI9XX2/Rx7zenzOS79Qg0t63mV0JVAHv\nT5n9tp+pu2/ta/uIYvsJ8H137zCzzxG0CC9Mc9so4+pxGfCguydS5kV5zAYyYt8vJZqQuy87yl3s\nAI5PeX0csIughtAUM4uF/5X2zD/quMyszsxmufvu8A9j/QC7uhR42N27Uva9O5zsMLP/BG4YybjC\nrincfZuZPQWcATxElo+XmZUBPwO+FHYp9Ox72MerD/19X/paZ4eZxYDJBF0h6WwbZVyY2TKC5P1+\nd+/omd/PzzRTfzQHjc3d96a8/A7wTynbXtBr26dGKq4UlwGfT50R8TEbSH9xZ/xYqessc54H5lkw\nYqqQ4Av1qAdn154kOD8CsAJIp4WUjkfD/aWz37f1C4d/bHvOi1wC9Dk6JYq4zGxqT9eTmVUA5wKv\nZvt4hT+7hwn6rn/Ua1kmj1ef35cB4v04sCY8Po8Cl1kwKu0EYB7wu6OIZUhxmdkZwLeBi929PmV+\nnz/TDMWVbmyzUl5eDMTD6V8CHwhjnAp8gLe27iONK4xtAcHJ9d+kzIv6mA3kUeBPw9Fn7wEOhv9M\nZf5YRTHaYaw9gI8SZPkOoA74ZTj/GOCxlPU+DLxG8N/ITSnzTyT4Q7AF+BFQlKG4pgHVwObwuTyc\nXwXcnbLeXGAnkNdr+zXARoI/mP8NlI5UXMB7w/d+KXy+OheOF3Al0AWsT3ksjuJ49fV9IeiKuzic\nLg4//5bweJyYsu1N4XabgD/I8Pd9sLhWh78HPcfn0cF+piMY21eBV8IYngQWpmz7mfBYbgE+PZJx\nha//Hri113aRHTOCfyx3h9/nHQTn0z4HfC5cbsC3wpg3kjKiNtPHSpUBREQkUuo6ExGRSCnRiIhI\npJRoREQkUko0IiISKSUaERGJlBKNyAgxs5aj3P7B8OpxzKzUzL5tZlstqDS91szebWaF4bQuxpac\noUQjMgqY2akEFZq3hbPuJqgSMM/dTyWoWlzhQVHHauCTfe5IJAuUaERGWHgl9tfM7GUz22jhPW/M\nLM/M/j1sofzUzB4zs54KCVcQVjIws5OAdxOUyElCUL7E3X8Wrvs/4foiOUHNa5GR98fAYuB0oAJ4\n3szWEpQfmUtQ/HQGQfmUe8NtzuVICaFTgfX+1sKMqV4Gzo4kcpFhUItGZOSdR1BhOOHudcDTBInh\nPOBH7p509z0EJVR6zAIa0tl5mIA6zWxShuMWGRYlGpGR11cZ9oHmA7QR1D6DoJbX6WY20O9vEdA+\njNhEMk6JRmTkrQU+aWb5Zjad4Ja7vwN+RXDTrjwzq+StpdrjwMkAHtyrpAa4OawkjZnNM7Pl4fQ0\noMFTbgkhkk1KNCIj72FgA0HF3jXAX4VdZQ8RVNl9maAM/28J7qoJwf1xLkjZx2cJbvC2Jbxh3Hc4\ncg+UJcBj0X4EkfSperNIDjGzUndvCVslvwPOdfc9ZjaB4JzNuQMMAujZx4+BG9190wiELDIojToT\nyS0/NbMpBLeLviVs6eDubWb2dwT3bn+zv43DG2/9j5KM5BK1aEREJFI6RyMiIpFSohERkUgp0YiI\nSKSUaEREJFJKNCIiEiklGhERidT/B8INL/qFJiDSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x142436d8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 1\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "print(np.mean(lrcv_L1.scores_[1],axis = 0))\n",
    "\n",
    "scores[0][:] = np.mean(lrcv_L1.scores_[1],axis = 0)\n",
    "    \n",
    "mse_mean = np.mean(scores, axis = 0)\n",
    "plt.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "plt.xlabel('log(C)')\n",
    "plt.ylabel('neg-logloss')\n",
    "plt.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.44755728,  1.12809827, -0.24050707,  0.03115452, -0.0763311 ,\n",
       "         0.61425104,  0.37697351,  0.11844018]])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best C  is: [1.1]\n"
     ]
    }
   ],
   "source": [
    "print ('best C  is:',lrcv_L1.C_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LogisticRegressionCV + L1正则 最优C选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for classifier LogisticRegressionCV(Cs=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 4.0, 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5.0, 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7, 5.8, 5.9, 6.0, 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8, 6.9, 7.0, 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 7.7, 7.8, 7.9, 8.0, 8.1, 8.2, 8.3, 8.4, 8.5, 8.6, 8.7, 8.8, 8.9, 9.0, 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7, 9.8, 9.9],\n",
      "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
      "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
      "           n_jobs=1, penalty='l1', random_state=None, refit=True,\n",
      "           scoring='neg_log_loss', solver='liblinear', tol=0.0001,\n",
      "           verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.80      0.87      0.83       101\n",
      "          1       0.70      0.58      0.63        52\n",
      "\n",
      "avg / total       0.77      0.77      0.77       153\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[88 13]\n",
      " [22 30]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\learning_curve.py:22: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the functions are moved. This module will be removed in 0.20\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "#使用测试集对模型进行预测\n",
    "from sklearn import *\n",
    "pre_test_logis_L1=lrcv_L1.predict(X_test)\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\"\n",
    "      % (lrcv_L1, metrics.classification_report(y_test, pre_test_logis_L1)))\n",
    "print(\"Confusion matrix:\\n%s\" % metrics.confusion_matrix(y_test, pre_test_logis_L1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2, 0.21, 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3, 0.31, 0.32, 0.33, 0.34, 0.35, 0.36, 0.37, 0.38, 0.39, 0.4, 0.41, 0.42, 0.43, 0.44, 0.45, 0.46, 0.47, 0.48, 0.49, 0.5, 0...85, 99.86, 99.87, 99.88, 99.89, 99.9, 99.91, 99.92, 99.93, 99.94, 99.95, 99.96, 99.97, 99.98, 99.99],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=1, penalty='l2', random_state=None, refit=True,\n",
       "           scoring='neg_log_loss', solver='liblinear', tol=0.0001,\n",
       "           verbose=0)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "#Cs = [1, 10,100,1000]\n",
    "Cs=[i/100 for i in range(1,10000)]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs，为了和GridSeachCV比较，也用liblinear\n",
    "\n",
    "lr_cv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lr_cv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[-0.53085928, -0.5154448 , -0.51263002, ..., -0.55006291,\n",
       "         -0.55006291, -0.55006292],\n",
       "        [-0.53323282, -0.50655638, -0.4947712 , ..., -0.46740091,\n",
       "         -0.4674009 , -0.4674009 ],\n",
       "        [-0.53725593, -0.51061104, -0.49889471, ..., -0.47198193,\n",
       "         -0.47198193, -0.47198193],\n",
       "        [-0.54129636, -0.51715397, -0.50696413, ..., -0.48741549,\n",
       "         -0.48741549, -0.48741549],\n",
       "        [-0.5027844 , -0.47052038, -0.45624181, ..., -0.41978853,\n",
       "         -0.41978853, -0.41978853]])}"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.52908576 -0.50405731 -0.49390037 ... -0.47932995 -0.47932995\n",
      " -0.47932995]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmYXHWd7/H3p7vTabKSjSQsnRBA\nlhhZbBCMCwqMDMoyjqPOqAMKRnF8HJ+ZOxovXh2vzjXqlXudmeuMAcE4ehkcFYksM7IzwgXSYDaS\nYBY6BLqzkc6e3qq+949zOnSa7k51d1VXV9fn9Tz11Fl+p863DqG+fX7nnN9XEYGZmdlgVRQ7ADMz\nGxmcUMzMLC+cUMzMLC+cUMzMLC+cUMzMLC+cUMzMLC+cUMzMLC+cUMzMLC+cUMzMLC+qih3AUJo6\ndWrMnj272GGYmZWUZ599dmdETDtau7JKKLNnz6a+vr7YYZiZlRRJm3Np5y4vMzPLCycUMzPLCycU\nMzPLCycUMzPLCycUMzPLCycUMzPLCycUMzPLi7J6DsVKW0RwoC1D84E2mg+2setAG/taOmjtyNLa\nkaG1PUtbJktrexaACkFFhZBAiApBZYUYXVVBzahKakZVcsyoSo6prqRmVMUR82NHVzGuuoqKChX5\nW5uVDicUG1YigsY9Laxr2su6rfvYsusgLzcfYkvzQZp2t9CWyQ5pPONGVyWvmuR9fE3ySpaPYlxN\nFRNqXt+mc9246irGjq6kqtKdATbyOaFYUbW0Z3jupWae2vgqT7+4i7VNe9nb0nF4/dRxozlx0jHM\nO2Ei75k7gyljq5k0tprJY5L3CTVV1IyqZHRVBdVVFYyuqqS6qgIB2QiykbxH+p6JoLU9S0t7hpb2\nDIfaMxxqy9DSkU3e02X7WzrY19rB/pYO9re2s7+1g30tyatpT0uyvqWdA22ZnL7n6KqKw0lnbHWS\neMaOTs6ExqfLxnZJXsl05RHLO8+gRo+qYHRVBZLPnmx4cUKxIffizgPcv7qJ367fSf3mZto6slQI\n3njCRK48+3jOmDmBs2aO5w3TxzO+ZtSA95OklR7UDPgjXyeTDQ60dSaeJMnsS6cPpEnoQGsmaZMu\n62y7Y38rDa8eZH+67FB7bskJQIKaqqR7rjPJHNOlG69rF15N2o1XXVnBqMok8Y6qFNVVFYeXjUqn\nq6uUtOmy7Iht0vnKSlEpUVkhqiqSdyc4K1pCkTQZuBOYDTQAH4yI5l7aTgDWAndFxGfTZX8K/Fcg\ngEbgoxGxs/CR20C8ur+VXzz3Mnf9rpG1TXsBOGPGeD524SwumjOFC+ZMZsIgkkexVFaICTWj8hJ7\nZ3I60C0Z7W/t4GBbBy3tWQ6lZ1ZHnmFlaenI0NKWoaUjw8G2DnYd6HYW1p6hrSNLNvLwpXtRIaiq\nqDicZCq6JJvXz1f0uL5SoqICKtLkVKHk2pe6v5O0S+aF0v1XJBfMDm9XofQaWtft1GU7Hbmd0j9C\nOnNjZ4p8bb6H9elMTm27zndLwD1td7TPfG3b3mJ4bf0155zAxDGF/X+smGcoC4GHImKRpIXp/Bd7\naft14LHOGUlVwPeAsyJip6RvA58F/rawIVt/1Tfs4vYnGvjNmq20Z4JzTjqWL7/3TK6YN5Pjjz2m\n2OENK/lMTr3JZIP2TJbWjiztmfTVEbRlMrR1JOvaMlnaO5L3to4s7Zl0ebqsPZMlk43kFUEmE3R0\nnc8GHZkgk812m0/Wd2Rf2ybbOZ/N0pEJOrJZIvNad2WQXFfLRpDNHjkf3bozAw63o3O683Oiy3ZA\nNtvDdp3JNjrfkomIIxYT6YLosq4UzD916ohOKFcDF6fTS4BH6SGhSHozMB34d6Cuc3H6GivpVWAC\nsKGw4VquIoJHf7+Df3pkI8807OLYMaP484tm8+HzT+K06eOLHV5Zq6wQlRVJN5jl3+Fk00cSOnL9\nke3psj6XhHbEtkdJhBNqCv9zX8yEMj0imgAioknScd0bSKoAvgt8DLikc3lEtEu6EVgFHADWA38x\nJFFbn57d3MzX71nD8i27OX5iDV+98iw+fH4tx1T7B8xGvsNdT6+7nFQe15cKmlAkPQjM6GHVTTl+\nxGeA+yJiS9f+RkmjgBuBc4FNwD8AXwK+0UMMC4AFALW1tf0J3/rhld2H+Nb961i6opHjxo9m0fvn\n8f7zTqS6yrfLmpWLgiaUiLi0t3WStkmamZ6dzAS299DsIuDtkj4DjAOqJe0HfpF+/sb0s35Gcg2m\npxgWA4sB6urqSqjHszRks8G/PLWZb96/lgj43LtP5VPvPIWxo30DoVm5Keb/9UuBa4FF6fvd3RtE\nxEc6pyVdB9RFxEJJxwNnSZoWETuAy0juArMh1Lj7EF/4+Up+u2En7zp9Gt/4o3mc4AvtZmWrmAll\nEfAzSdcDLwF/AiCpDvh0RNzQ24YR0Sjpa8DjktqBzcB1hQ/ZOt29/BW+/KvVZLLBN98/jw+ff5Kf\nQzArc4pSuu9tkOrq6sI15Qcnkw0W3b+WW/7zRepmTeK7HzybWVPGFjssMysgSc9GRN3R2rmj23K2\nv7WDv7zjdzy0bjvXvXU2X37vmR6jyswOc0KxnLzcfJAbltSzfvt+vn71XD520exih2Rmw4wTih3V\n6lf2cN3ty2jtyPCjj5/P20+bVuyQzGwYckKxPq3bupeP/vBpxlZX8a8LLuTU48YVOyQzG6acUKxX\nG7bv56O3Pk1NVSV3fPJCaqeMKXZIZjaM+Yqq9Wjzqwf4yK1PAeKnn3yLk4mZHZUTir3OK7sP8We3\nPE1bR5af3vAWTpnmbi4zOzp3edkRtu9t4c9ueYq9Le3c8ckLOX2GRwc2s9z4DMUOa+vI8qmfPMuO\nfa0s+cQFvPGEicUOycxKiM9Q7LD/cd9afvfSbr7/kfM4r3ZSscMxsxLjMxQDYOmKRn70ZAPXv+1k\nrpg3s9jhmFkJckIxNmzfx8JfrKRu1iQW/uEZxQ7HzEqUE0qZO9Dawad/8hxjqiv5xz87j1Eem8vM\nBsjXUMpYRPClX65i0479/OT6tzBjYk2xQzKzEuY/R8vYvzy1maUrGvnrPzidt546tdjhmFmJc0Ip\nU8u37Obr96zhkjOO48Z3nlLscMxsBHBCKUPtmSxf/PlKjhtfw80fPIeKCldaNLPBc0IpQ0uebOCF\nbfv46pVnMXHMqGKHY2YjhBNKmdm2t4X//eB63nX6NC47a3qxwzGzEcQJpcz83b1ractk+dur5iK5\nq8vM8scJpYw8uWEnS1c0cuM7T2HWlLHFDsfMRhgnlDLR1pHlK0ufp3byGG682Hd1mVn++cHGMnHb\nEy+yYft+bruujppRlcUOx8xGIJ+hlIHG3Yf43oPrueys6bz7DF+IN7PCcEIpA9+4dw1B8JX3nVXs\nUMxsBHNCGeEe//0O7lu1lc++61ROmuy68GZWOE4oI1hrR4avLn2ek6eO5ZPvmFPscMxshPNF+RHs\nlsc38eLOA/z4ExcwusoX4s2ssHyGMkLtOdjO9x/dyOVzZ/CON0wrdjhmVgacUEaonzy9mYNtGT5/\n2WnFDsXMykRREoqkyZIekLQ+fZ/UR9sJkl6R9I9dln1I0kpJz0v69tBEXTpa2jPc/sSLXHz6NM6Y\nMaHY4ZhZmSjWGcpC4KGIOA14KJ3vzdeBxzpnJE0BvgNcEhFzgemSLilksKXml8+9ws79bXzqHX4i\n3syGTrESytXAknR6CXBNT40kvRmYDvymy+I5wO8jYkc6/yDwxwWKs+RkssHixzdy9okTuXDO5GKH\nY2ZlpFgJZXpENAGk78d1byCpAvgu8DfdVm0AzpA0W1IVSTI6qcDxlozfPL+VhlcP8ql3nuLRhM1s\nSBXstmFJDwIzelh1U44f8RngvojY0vWHMSKaJd0I3AlkgSdJzlp6i2MBsACgtrY2x12Xpojgnx/b\nyKwpY3jP3J4OvZlZ4RQsoUTEpb2tk7RN0syIaJI0E9jeQ7OLgLdL+gwwDqiWtD8iFkbEr4Ffp5+1\nAMj0EcdiYDFAXV1dDPwbDX9Pv7iLFS/v4RvXvJFKl/U1syFWrC6vpcC16fS1wN3dG0TERyKiNiJm\nA/8F+HFELASQdFz6PonkTObWoQh6uPvBYxuZMraaD7z5xGKHYmZlqFgJZRFwmaT1wGXpPJLqJOWS\nHL4naQ3wBLAoIn5fuFBLw7qte3nkhR1c99bZHp7ezIqiKEOvRMSrwOtu9Y2IeuCGHpb/CPhRl/k/\nLWB4JWnx45s4ZlQlH7toVrFDMbMy5SflR4DG3YdYuryRD19wEseOqS52OGZWppxQRoDbfvsiAVz/\ntpOLHYqZlTEnlBK352A7dzzzEle+aSYnTnK9EzMrHieUEveTpzdzoC3DAg+zYmZF5oRSwpJBIBt4\nxxumcdbxHgTSzIrLCaWE3fW7V9i5v5VPuxqjmQ0DTiglKpMNbnl8E/NOmMhFp0wpdjhmZk4opeqB\nNdvYtPMAn3rnHA8CaWbDghNKCeocBLJ28hgu9yCQZjZMOKGUoPrNzSzfsptPvv1kqir9n9DMhgf/\nGpWgf6vfwrjRVXzgzS4DY2bDhxNKiWnryPIfz2/jsrOmc0y1B4E0s+HDCaXEPLFhJ3sOtfPeeTOL\nHYqZ2RGcUErMr1c2Mr6mire/YWqxQzEzO4ITSglpac/wwPPbeM/cGYyucneXmQ0vTigl5PHf72Bf\nawfve5O7u8xs+HFCKSH3rmri2DGjmH+qu7vMbPhxQikRLe0ZHlyzjcvnzmCUnz0xs2HIv0wl4pF1\n2znQluF9bzq+2KGYmfXICaVE3LOyiSljq7lwzuRih2Jm1iMnlBJwoLWDh9Zt4w/nzfBQK2Y2bPnX\nqQQ8vG47Le1Z3jvP3V1mNnzllFAkzZc0Np3+qKSbJc0qbGjW6Z6VjUwbP5oLTnZ3l5kNX7meofwT\ncFDS2cAXgM3AjwsWlR22r6WdR17YwXvnzaSywnVPzGz4yjWhdEREAFcD34uI7wHjCxeWdXpw7Tba\nOrJ+mNHMhr2qHNvtk/Ql4KPAOyRVAqMKF5Z1umdFEzMn1nBe7aRih2Jm1qdcz1A+BLQC10fEVuAE\n4DsFi8oA2HOoncfX7+CKeTOpcHeXmQ1zOZ+hkHR1ZSS9ATgDuKNwYRnAb57fSnsm3N1lZiUh1zOU\nx4HRkk4AHgI+DvyoUEFZ4p6VTZw46RjOOenYYodiZnZUuSYURcRB4P3AP0TEHwFzCxeWNR9o44kN\nO3nvm2YiubvLzIa/nBOKpIuAjwD3pssGXJBD0mRJD0han773eMVZUkbS8vS1tMvykyU9nW5/p6Tq\ngcYyXP3H81vpyAZXeuwuMysRuSaUzwNfAu6KiOclzQEeGcR+FwIPRcRpJF1oC3tpdygizklfV3VZ\n/i3gf6XbNwPXDyKWYemelU3MmjKGucdPKHYoZmY5ySmhRMRj6Q/69yWNi4hNEfG5Qez3amBJOr0E\nuCbXDZX0/7wb+PlAti8FO/e38uTGnbzP3V1mVkJyHXplnqTfAauBNZKelTSYayjTI6IJIH0/rpd2\nNZLqJT0lqTNpTAF2R0RHOv8yyW3MI8b9q7eSDTxUvZmVlFxvG/4B8FcR8QiApIuBW4C39raBpAeB\nGT2suqkf8dVGRGPaxfawpFXA3h7aRR9xLAAWANTW1vZj18Vzz4pGTpk2ljNmeDACMysduSaUsZ3J\nBCAiHu0cLLI3EXFpb+skbZM0MyKaJM0EtvfyGY3p+yZJjwLnAr8AjpVUlZ6lnAg09hHHYmAxQF1d\nXa+JZ7jYvreFZxp28bl3n+buLjMrKblelN8k6b9Jmp2+vgy8OIj9LgWuTaevBe7u3kDSJEmj0+mp\nwHxgTTqm2CPAB/ravlTdt6qJCPwwo5mVnFwTyieAacAvgbvS6Y8PYr+LgMskrQcuS+eRVCfp1rTN\nmUC9pBUkCWRRRKxJ130R+CtJG0iuqfxwELEMK/esbOL06eM5bbq7u8ystOTU5RURzcBg7urq/nmv\nApf0sLweuCGdfhKY18v2m4AL8hXPcNG4+xD1m5v568veUOxQzMz6rc+EIunX9HHBu9uzITZI961q\nAuB9Z/vuLjMrPUc7Q/mfQxKFAUl319zjJ3Dy1D7vdzAzG5b6TCgR8dhQBVLutuw6yPItu/nC5acX\nOxQzswHJ6RpK+vxH966vPUA98I30mogNwr2d3V3z3N1lZqUp1+dQ7gcywP9N5z8MiCSp/Ai4Mu+R\nlZl7VjZy9okTqZ0yptihmJkNSK4JZX5EzO8yv0rSExExX9JHCxFYOWnYeYDVr+zlpivOLHYoZmYD\nlutzKOMkvaVzRtIFwLh0tqPnTSxXnd1dV/hhRjMrYbmeodwA3CZpHElX117g+nT4lW8WKrhy8esV\njZxXeywnHHtMsUMxMxuwXB9sXAbMkzSRpHrj7i6rf1aQyMrEhu37Wbd1H19531nFDsXMbFByHb5+\noqSbSYphPSjpu2lysUG6Z2UjErzX3V1mVuJyvYZyG7AP+GD62gvcXqigyslvnt/G+bMmM31CTbFD\nMTMblFyvoZwSEX/cZf5rkpYXIqBysudQO2u37uXzl3jsLjMrfbmeoRyS9LbOGUnzgUOFCal8PPdS\nMxFw/uxJxQ7FzGzQcj1DuRFY0nlRHtgFXFeooMpFfcMuKivEObXHFjsUM7NBy/Uur+XA2ZImpPM9\nleG1flrW0Mwbj5/AmOpc87qZ2fB1tOHr/6qX5QBExM0FiKkstHZkWL5lN39+4axih2JmlhdH+9PY\nZQMLZPUre2jryFI3e3KxQzEzy4ujDV//taEKpNwsa2gGoM4X5M1shMj1Lq/DJD1XiEDKTX3DLuZM\nHcvUcaOLHYqZWV70O6GQ3OVlg5DNBssamjnf3V1mNoIMJKHcm/coysyGHfvZc6jd3V1mNqL0O6FE\nxJcLEUg5WdawC8BnKGY2ouRaAngfvZcA/uuI2JTvwEay+oZmpo4bzSxXZzSzESTXJ+puBhpJSgCL\npATwDOAFkoEjLy5EcCPVsoZdnD970uHneczMRoJcu7wuj4gfRMS+iNgbEYuBKyLiTsAXAvqhac8h\nXm4+5O4uMxtxck0oWUkflFSRvj7YZV33rjDrQ+fzJ04oZjbS5JpQPgJ8DNgObEunPyrpGOCzBYpt\nRKpv2MWY6krOnOlBCMxsZMl1cMhNwJW9rP5t/sIZ+ZY1NHNe7SSqKgdyx7aZ2fCVawngN0h6SNLq\ndP5Nknz7cD/tOdTOuq173d1lZiNSrn8m3wJ8CWgHiIiVJHd6WT+4oJaZjWS5JpQxEfFMt2UdA92p\npMmSHpC0Pn3v8RdWUkbS8vS1tMvyz0raICkkTR1oHEPNBbXMbCTLNaHslHQK6R1dkj4ANA1ivwuB\nhyLiNOChdL4nhyLinPR1VZflTwCXApsHEcOQc0EtMxvJck0ofwH8ADhD0ivA54FPD2K/VwNL0ukl\nwDX92TgifhcRDYPY/5Br7ciwYstu1z8xsxEr14TyCnA78HfAvwIPANcOYr/TI6IJIH0/rpd2NZLq\nJT0lqV9JZ7hZ/coeWjuyviBvZiNWrn0vdwO7gedIhmA5KkkPkgzP0t1NOe4ToDYiGiXNAR6WtCoi\nNvZjeyQtABYA1NbW9mfTvHJBLTMb6XJNKCdGxOX9+eCIuLS3dZK2SZoZEU2SZpI8MNnTZzSm75sk\nPQqcC/QroaTDxCwGqKurK9pT/S6oZWYjXa5dXk9KmpfH/S7ltS6za0nOgI4gaZKk0en0VGA+sCaP\nMQyZbDao39zssxMzG9FyTShvA56V9IKklZJWSVo5iP0uAi6TtB64LJ1HUp2kW9M2ZwL1klYAjwCL\nImJN2u5zkl4GTgRWdtlmWNqwYz+7D7b7+omZjWi5dnn9YT53GhGvApf0sLweuCGdfhLo8awoIv4e\n+Pt8xlRILqhlZuUg17G8Sup5j+HGBbXMrBx4hMIh4IJaZlYOnFAKrLOglh9oNLORzgmlwDqfP7nA\nCcXMRjgnlAJzQS0zKxdOKAXmglpmVi78K1dAe1uSglp+oNHMyoETSgE9tzkpqOXrJ2ZWDpxQCmiZ\nC2qZWRlxQikgF9Qys3LihFIgLqhlZuXGCaVAVr+yNy2o5QvyZlYenFAKpHNASJ+hmFm5cEIpEBfU\nMrNy44RSAC6oZWblyAmlADamBbXc3WVm5cQJpQA8IKSZlSMnlAJY1rDLBbXMrOw4oRSAC2qZWTly\nQskzF9Qys3LlhJJn9en1Ez/QaGblxgklz5alBbXOmjmh2KGYmQ0pJ5Q8c0EtMytX/tXLIxfUMrNy\n5oSSR50Ftc73BXkzK0NOKHlU39BMZYU41wW1zKwMOaHk0TMNu1xQy8zKlhNKnriglpmVOyeUPHFB\nLTMrd04oeVKfFtR68yyfoZhZeSpKQpE0WdIDktan7z3+WS8pI2l5+lraZflPJb0gabWk2ySNGrro\ne7asoZk5U8cybbwLaplZeSrWGcpC4KGIOA14KJ3vyaGIOCd9XdVl+U+BM4B5wDHADQWN9iiSglq7\n/PyJmZW1YiWUq4El6fQS4Jr+bBwR90UKeAY4Mc/x9YsLapmZFS+hTI+IJoD0/bhe2tVIqpf0lKTX\nJZ20q+tjwL8XLtSjW3Z4QEgnFDMrXwV7YELSg8CMHlbd1I+PqY2IRklzgIclrYqIjV3Wfx94PCL+\ns484FgALAGpra/ux69zVpwW1ZruglpmVsYIllIi4tLd1krZJmhkRTZJmAtt7+YzG9H2TpEeBc4GN\n6Wd8FZgGfOoocSwGFgPU1dXFAL7KUT3jglpmZkXr8loKXJtOXwvc3b2BpEmSRqfTU4H5wJp0/gbg\nPcCfRkR2SCLuhQtqmZklipVQFgGXSVoPXJbOI6lO0q1pmzOBekkrgEeARRGxJl33z8B04P+ltxR/\nZWjDf40LapmZJYoy6FREvApc0sPyetJbgCPiSZLbgnvaftgMllXvglpmZoCflB80F9QyM0v4V3AQ\n9ra0s9YFtczMACeUQXFBLTOz1zihDEJnQa1zTnJBLTMzJ5RBWJYW1Bo7etjcI2BmVjROKAPU2pFh\nuQtqmZkd5oQyQC6oZWZ2JCeUAXJBLTOzIzmhDNCyhmZOdkEtM7PDnFAGIJsNnt28y91dZmZdOKEM\nwMYd+2l2QS0zsyM4oQyAC2qZmb2eE8oAJAW1ql1Qy8ysCyeUAVi2eRfnz57sglpmZl04ofTT1j0t\nbNnlglpmZt05ofTTsvT5E9/hZWZ2JCeUfnJBLTOznjmh9NOyhmbOrT3WBbXMzLrxr2I/7G1pZ93W\nvb5d2MysB04o/fDc5mayLqhlZtYjJ5R+cEEtM7PeOaH0w7KGXcx1QS0zsx45oeSorSPL8i273d1l\nZtYLJ5QcrW7c44JaZmZ9cELJ0bIXXVDLzKwvTig5ckEtM7O+OaHkoLOgVt0sd3eZmfXGCSUHm3Ym\nBbXOP9ndXWZmvXFCycEzL7qglpnZ0Tih5MAFtczMjq4oT+hJmgzcCcwGGoAPRkRzD+0ywKp09qWI\nuCpd/kOgDhDwe+C6iNhfqHhPOW4c0yfWuKCWmVkfFBFDv1Pp28CuiFgkaSEwKSK+2EO7/RExrofl\nEyJibzp9M7A9IhYdbb91dXVRX1+fh29gZlY+JD0bEXVHa1esLq+rgSXp9BLgmv5s3CWZCDgGGPqs\naGZmRyhWQpkeEU0A6ftxvbSrkVQv6SlJRyQdSbcDW4EzgH/obUeSFqSfUb9jx448hW9mZt0V7BqK\npAeBGT2suqkfH1MbEY2S5gAPS1oVERsBIuLjkipJksmHgNt7+oCIWAwshqTLqz/fwczMclewhBIR\nl/a2TtI2STMjoknSTGB7L5/RmL5vkvQocC6wscv6jKQ7gb+hl4RiZmZDo1hdXkuBa9Ppa4G7uzeQ\nNEnS6HR6KjAfWKPEqelyAVcC64YkajMz61WxCnssAn4m6XrgJeBPACTVAZ+OiBuAM4EfSMqSJL5F\nEbFGUgWwRNIEktuGVwA3FuNLmJnZa4qSUCLiVeCSHpbXAzek008C83pokyU5WzEzs2HET8qbmVle\nFOXBxmKRtAPYPMDNpwI78xhOvjiu/nFc/eO4+mekxjUrIqYdrVFZJZTBkFSfy5OiQ81x9Y/j6h/H\n1T/lHpe7vMzMLC+cUMzMLC+cUHK3uNgB9MJx9Y/j6h/H1T9lHZevoZiZWV74DMXMzPLCCaUXkr4j\naZ2klZLuknRsL+0ul/SCpA1pbZdCx/Unkp6XlE1HFuitXYOkVZKWSyp4EZh+xDXUx2uypAckrU/f\nJ/XSLpMeq+WSlhYwnj6/v6TRku5M1z8taXahYulnXNdJ2tHlGN0wRHHdJmm7pNW9rJekv0/jXinp\nvGEQ08WS9nQ5Vl8pdEzpfk+S9Iikten/i3/ZQ5vCHq+I8KuHF/AHQFU6/S3gWz20qSQZrHIOUE0y\nDMxZBY7rTOB04FGgro92DcDUITxeR42rSMfr28DCdHphT/8d03X7h+AYHfX7A58B/jmd/jBw5zCJ\n6zrgH4fq31OX/b4DOA9Y3cv6K4D7SYZhuhB4ehjEdDFwTxGO1UzgvHR6PEk12+7/HQt6vHyG0ouI\n+E1EdKSzTwEn9tDsAmBDRGyKiDbgX0mKhxUyrrUR8UIh9zEQOcY15MeLQRZzy7Ncvn/XeH8OXJIO\nglrsuIoiIh4HdvXR5Grgx5F4Cjg2HcG8mDEVRUQ0RcRz6fQ+YC1wQrdmBT1eTii5+QRJVu/uBGBL\nl/mXef1/wGIJ4DeSnpW0oNjBpIpxvAZdzC2Pcvn+h9ukf9DsAaYUKJ7+xAXwx2k3yc8lnVTgmHI1\nXP8fvEjSCkn3S5o71DtPu0rPBZ7utqqgx6tYow0PC+qjCFhE3J22uQnoAH7a00f0sGzQt83lElcO\n5kdSnOw44AFJ69K/rIoZ15Afr358TK/F3PIol+9fkGN0FLns89fAHRHRKunTJGdR7y5wXLkoxvE6\nmudIhirZL+kK4FfAaUO1c0njgF8An4+0XHrX1T1skrfjVdYJJfooAgYg6VrgfcAlkXZAdvMy0PUv\ntROBxkLHleNndBYn2y7pLpJujUEllDzENeTHS3kq5pYnuXz/zjYvS6oCJlL47pWjxhXJCOGdbiG5\nrjgcFOTf1GB0/RGPiPskfV/AHlY9AAADZElEQVTS1Igo+BhfkkaRJJOfRsQve2hS0OPlLq9eSLoc\n+CJwVUQc7KXZMuA0SSdLqia5iFqwO4RyJWmspPGd0yQ3GPR4R8oQK8bxGnAxtwLEksv37xrvB4CH\ne/ljZkjj6tbPfhVJ//xwsBT48/TupQuBPZ1dnMUiaUbndS9JF5D8zr7a91Z52a+AHwJrI+LmXpoV\n9ngN9Z0IpfICNpD0NS5PX5133hwP3Nel3RUkd1NsJOn6KXRcf0TyV0YrsA34j+5xkdytsyJ9PT9c\n4irS8ZoCPASsT98np8vrgFvT6bcCq9LjtQq4voDxvO77A/+d5A8XgBrg39J/f88Acwp9jHKM65vp\nv6UVwCPAGUMU1x1AE9Ce/vu6Hvg0SSE+SLpw/k8a9yr6uPNxCGP6bJdj9RTw1iE6Vm8j6b5a2eV3\n64qhPF5+Ut7MzPLCXV5mZpYXTihmZpYXTihmZpYXTihmZpYXTihmZpYXTihmeSRp/yC3/3n6tD6S\nxkn6gaSN6eixj0t6i6TqdLqsH0y24ccJxWyYSMd8qoyITemiW0mekj8tIuaSjPg7NZIBHB8CPlSU\nQM164YRiVgDpk8jfkbRaSV2aD6XLK9KhOJ6XdI+k+yR9IN3sI6RP8ks6BXgL8OWIyEIyLExE3Ju2\n/VXa3mzY8CmzWWG8HzgHOBuYCiyT9DjJsC6zgXkkIx+vBW5Lt5lP8hQ2wFxgeURkevn81cD5BYnc\nbIB8hmJWGG8jGZ03ExHbgMdIEsDbgH+LiGxEbCUZxqTTTGBHLh+eJpq2zjHbzIYDJxSzwuitKFZf\nxbIOkYzlBclYUGdL6uv/0dFAywBiMysIJxSzwngc+JCkSknTSMrGPgP8lqRQVYWk6STlYjutBU4F\niKQWSz3wtS4j154m6ep0egqwIyLah+oLmR2NE4pZYdxFMurrCuBh4AtpF9cvSEaoXQ38gKSi3p50\nm3s5MsHcQFI4bIOkVSR1SDprV7wLuK+wX8GsfzzasNkQkzQukmp+U0jOWuZHxFZJx5BcU5nfx8X4\nzs/4JfCliHhhCEI2y4nv8jIbevdIOhaoBr6enrkQEYckfZWkxvdLvW2cFsH6lZOJDTc+QzEzs7zw\nNRQzM8sLJxQzM8sLJxQzM8sLJxQzM8sLJxQzM8sLJxQzM8uL/w9SozEu8elDRAAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14f35748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best C  is: [0.28]\n"
     ]
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 1\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "print(np.mean(lr_cv_L2.scores_[1],axis = 0))\n",
    "\n",
    "scores[0][:] = np.mean(lr_cv_L2.scores_[1],axis = 0)\n",
    "    \n",
    "mse_mean = np.mean(scores, axis = 0)\n",
    "plt.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "plt.xlabel('log(C)')\n",
    "plt.ylabel('neg-logloss')\n",
    "plt.show()\n",
    "\n",
    "print ('best C  is:',lr_cv_L2.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for classifier LogisticRegressionCV(Cs=[0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2, 0.21, 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3, 0.31, 0.32, 0.33, 0.34, 0.35, 0.36, 0.37, 0.38, 0.39, 0.4, 0.41, 0.42, 0.43, 0.44, 0.45, 0.46, 0.47, 0.48, 0.49, 0.5, 0...85, 99.86, 99.87, 99.88, 99.89, 99.9, 99.91, 99.92, 99.93, 99.94, 99.95, 99.96, 99.97, 99.98, 99.99],\n",
      "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
      "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
      "           n_jobs=1, penalty='l2', random_state=None, refit=True,\n",
      "           scoring='neg_log_loss', solver='liblinear', tol=0.0001,\n",
      "           verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.80      0.87      0.83       101\n",
      "          1       0.70      0.58      0.63        52\n",
      "\n",
      "avg / total       0.77      0.77      0.77       153\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[88 13]\n",
      " [22 30]]\n"
     ]
    }
   ],
   "source": [
    "#对测试集进行预测\n",
    "from sklearn import metrics\n",
    "pre_test_logis_L2=lr_cv_L2.predict(X_test)\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\"\n",
    "      % (lr_cv_L2, metrics.classification_report(y_test, pre_test_logis_L2)))\n",
    "print(\"Confusion matrix:\\n%s\" % metrics.confusion_matrix(y_test, pre_test_logis_L2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 支持向量机分类回归问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#LinearSVC不能得到每类的概率，在Otto数据集要求输出每类的概率，这里只是示意SVM的使用方法\n",
    "#https://xacecask2.gitbooks.io/scikit-learn-user-guide-chinese-version/content/sec1.4.html\n",
    "#1.4.1.2. 得分与概率\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "SVC1 = LinearSVC().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for classifier LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.79      0.87      0.83       101\n",
      "          1       0.69      0.56      0.62        52\n",
      "\n",
      "avg / total       0.76      0.76      0.76       153\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[88 13]\n",
      " [23 29]]\n"
     ]
    }
   ],
   "source": [
    "#在校验集上测试，估计模型性能\n",
    "y_predict = SVC1.predict(X_test)\n",
    "\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\"\n",
    "      % (SVC1, metrics.classification_report(y_test, y_predict)))\n",
    "print(\"Confusion matrix:\\n%s\" % metrics.confusion_matrix(y_test, y_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 线性SVM正则参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "线性SVM LinearSVC的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） \n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fit_grid_point_Linear(C, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC2 =  LinearSVC( C = C,penalty='l2')\n",
    "    SVC2 = SVC2.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC2.score(X_val, y_val)\n",
    "    \n",
    "    print(\"accuracy: {}\".format(accuracy),\"C=%f\"%(C))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.7777777777777778 C=0.010000\n",
      "accuracy: 0.7777777777777778 C=0.021544\n",
      "accuracy: 0.7777777777777778 C=0.046416\n",
      "accuracy: 0.7647058823529411 C=0.100000\n",
      "accuracy: 0.7647058823529411 C=0.215443\n",
      "accuracy: 0.7647058823529411 C=0.464159\n",
      "accuracy: 0.7647058823529411 C=1.000000\n",
      "accuracy: 0.7647058823529411 C=2.154435\n",
      "accuracy: 0.7712418300653595 C=4.641589\n",
      "accuracy: 0.7712418300653595 C=10.000000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XuUHOV55/HvDw1CgA0SaJABCSQh\nIaFpbs5YdnxZG4xZ2XEAc5WSzUISTG7gLF4nwcc5tsOenCTrc8LJOsQJ9uLcvPQIgbBsCxBGBMJZ\nwBpADrogaZAFyJJh8EoGIQndnv2jqu3W0DPTM9011Zff55w+3VX1Vs1TM9I8817qfRURmJmZjdYR\neQdgZmbNzYnEzMxq4kRiZmY1cSIxM7OaOJGYmVlNnEjMzKwmTiRmZlYTJxIzM6uJE4mZmdWkI+8A\nxsLkyZNj+vTpeYdhZtZUnn766dcionO4cm2RSKZPn05vb2/eYZiZNRVJL1ZTzk1bZmZWk0wTiaQF\nkjZI6pN0S4Xjt0lanb42StqZ7r+gbP9qSXslXZYe+6ikZ9L9j0ualeU9mJnZ0DJr2pI0Drgd+Biw\nFVglaVlErCuViYiby8rfBJyf7n8EOC/dfwLQB6xIi34NuDQi1kv6feBPgeuyug8zMxtaljWS+UBf\nRGyOiH1AEbh0iPKLgLsq7L8SuD8idqfbARyXfj4e2FaneM3MbBSy7Gw/FXi5bHsr8N5KBSWdDswA\nVlY4vBD467Lt64HlkvYArwPvq0u0ZmY2KlnWSFRh32CraC0ElkTEwcMuIJ0MnA08WLb7ZuATETEV\n+CaHJ5nyc2+Q1Cupt7+/f8TBm5lZdbJMJFuBaWXbUxm8GWohlZu1rgaWRsR+AEmdwLkR8VR6vAd4\nf6ULRsQdEdEdEd2dncMOgzYzs1HKsmlrFTBb0gzgxyTJ4tcGFpI0B5gEPFHhGouAz5dt7wCOl3Rm\nRGwk6chfX+/AS/71X2Hjxqyu3nw6OuB3fgemTMk7EjNrJJklkog4IOlGkmapccCdEbFW0q1Ab0Qs\nS4suAooxYPF4SdNJajSPDrjmp4F7JB0iSSy/ldU9FIuwfHlWV28+ETBuHHzhC3lHYmaNRAN+f7ek\n7u7u8JPttZs5E977XrirUiOkmbUcSU9HRPdw5fxku1WtqwvWrMk7CjNrNE4kVrVCAZ5/HvbtyzsS\nM2skTiRWtUIBDhyATZvyjsTMGokTiVWtUEje167NNw4zayxOJFa1OXOSUVvuJzGzck4kVrUJE2DW\nLCcSMzucE4mNSKHgRGJmh3MisREpFKCvD/bsyTsSM2sUTiQ2IoVC8oT788/nHYmZNQonEhuR0sgt\nN2+ZWYkTiY3IrFkwfrwTiZn9ghOJjUhHB8yd60RiZr/gRGIj5pFbZlbOicRGrFCAl16C11/POxIz\nawROJDZipQ73devyjcPMGoMTiY1YV1fy7uYtMwMnEhuF6dPhmGOcSMws4URiI3bEEUmtxLMAmxk4\nkdgoeeSWmZU4kdioFArwk5/Aa6/lHYmZ5c2JxEal1OHu5i0zyzSRSFogaYOkPkm3VDh+m6TV6Wuj\npJ3p/gvK9q+WtFfSZekxSfrztPx6SZ/J8h6sMs+5ZWYlHVldWNI44HbgY8BWYJWkZRHx86cPIuLm\nsvI3Aeen+x8Bzkv3nwD0ASvSotcB04C5EXFI0klZ3YMN7pRTYOJE10jMLNsayXygLyI2R8Q+oAhc\nOkT5RcBdFfZfCdwfEbvT7d8Dbo2IQwAR8WodY7YqSe5wN7NElonkVODlsu2t6b63kXQ6MANYWeHw\nQg5PMGcA10jqlXS/pNl1itdGqJRIIvKOxMzylGUiUYV9g/3KWQgsiYiDh11AOhk4G3iwbPdRwN6I\n6Aa+DtxZ8YtLN6TJpre/v3/Ewdvwurpgxw7Yvj3vSMwsT1kmkq0kfRklU4Ftg5QdWOsouRpYGhH7\nB1z3nvTzUuCcSheMiDsiojsiujs7O0cUuFXHHe5mBtkmklXAbEkzJI0nSRbLBhaSNAeYBDxR4RqV\n+k3uAy5MP38Y2Fi3iG1EPATYzCDDUVsRcUDSjSTNUuOAOyNiraRbgd6IKCWVRUAx4vCWdknTSWo0\njw649F8C35J0M7ALuD6re7ChdXbClCmukZi1O0Ub9JR2d3dHb29v3mG0pIsugjfegKeeyjsSM6s3\nSU+n/dFD8pPtVpPS5I2HDuUdiZnlxYnEalIowJtvwosv5h2JmeXFicRqUhq55Q53s/blRGI18WqJ\nZuZEYjU57jiYNs2JxKydOZFYzTznlll7cyKxmhUKsH49HDiQdyRmlgcnEqtZoQD79sELL+QdiZnl\nwYnEauY5t8zamxOJ1Wzu3GR9EicSs/bkRGI1O+YYOOMMJxKzduVEYnXhkVtm7cuJxOqiUIBNm+Ct\nt/KOxMzGmhOJ1UWhAAcPwoYNeUdiZmPNicTqwlOlmLUvJxKrizPPhI4OJxKzduREYnUxfjzMmeNE\nYtaOnEisbgoFTydv1o6cSKxuCgXYvDlZ6MrM2ocTidVNqcN93bp84zCzseVEYnXjObfM2pMTidXN\nzJkwYYITiVm7yTSRSFogaYOkPkm3VDh+m6TV6WujpJ3p/gvK9q+WtFfSZQPO/aqkXVnGbyMzbhzM\nm+cOd7N205HVhSWNA24HPgZsBVZJWhYRP29Bj4iby8rfBJyf7n8EOC/dfwLQB6woK9sNTMwqdhu9\nQgEefjjvKMxsLGVZI5kP9EXE5ojYBxSBS4covwi4q8L+K4H7I2I3/DxBfQX44zrHa3XQ1QU//jHs\n2JF3JGY2VrJMJKcCL5dtb033vY2k04EZwMoKhxdyeIK5EVgWEduH+uKSbpDUK6m3v79/RIHb6JU6\n3N28ZdY+skwkqrAvBim7EFgSEQcPu4B0MnA28GC6fQpwFfDV4b54RNwREd0R0d3Z2TmiwG30PHLL\nrP1kmUi2AtPKtqcC2wYpO7DWUXI1sDQi9qfb5wOzgD5JW4BjJPXVJ1yrh2nT4J3vdI3ErJ1k1tkO\nrAJmS5oB/JgkWfzawEKS5gCTgCcqXGMR8PnSRkR8D3hX2bm7ImJWneO2Gkhe5Mqs3WRWI4mIAyT9\nGQ8C64HFEbFW0q2SLikruggoRsRhzV6SppPUaB7NKkbLRlcXPPccxGANmWbWUrKskRARy4HlA/Z9\nccD2lwc5dwuDdM6XlXlHbRFaFgoF+MY34NVXYcqUvKMxs6z5yXarO3e4m7UXJxKrOw8BNmsvTiRW\ndyedBJMnu0Zi1i6cSKzupKTD3YnErD04kVgmSqsleuSWWetzIrFMFArw+uuwdWvekZhZ1pxILBMe\nuWXWPpxILBOlZXedSMxanxOJZWLSJDjlFCcSs3bgRGKZ8ZxbZu3BicQyUyjA+vVw8ODwZc2seTmR\nWGYKBdizB370o7wjMbMsOZFYZjxyy6w9OJFYZs46K3l3IjFrbU4klpl3vANmzHAiMWt1TiSWqdJU\nKWbWuqpKJJLukfQrkpx4bEQKBXj+edi3L+9IzCwr1SaGr5Gst75J0l9KmpthTNZCCgU4cAA2bco7\nEjPLSlWJJCK+HxG/Drwb2AI8JOn/SvpNSUdmGaA1N0+VYtb6qm6qknQicB1wPfAs8DckieWhTCKz\nljBnDowb50Ri1so6qikk6V5gLvAvwK9GxPb0UI+k3qyCs+Y3YQLMnu0Od7NWVm2N5G8jYl5E/EVZ\nEgEgIroHO0nSAkkbJPVJuqXC8dskrU5fGyXtTPdfULZ/taS9ki5Lj30rveYaSXe6aa3xec4ts9ZW\nbSI5S9LE0oakSZJ+f6gTJI0Dbgc+DswDFkmaV14mIm6OiPMi4jzgq8C96f5HyvZfCOwGVqSnfYuk\ndnQ2cDRJU5s1sK4u6OtLpksxs9ZTbSL5dETsLG1ExA7g08OcMx/oi4jNEbEPKAKXDlF+EXBXhf1X\nAvdHxO70ay+PFPADYGqV92A5KRSSJXfXr887EjPLQrWJ5AhJKm2ktY3xw5xzKvBy2fbWdN/bSDod\nmAGsrHB4IRUSTNqk9RvAA8PEYTnznFtmra3aRPIgsFjSRyVdSPKLfbhf4KqwLwYpuxBYEhGHTTgu\n6WSSJqwHK5zzd8BjEfHvFb+4dIOkXkm9/f39w4RqWZo1C8aPd4e7WauqNpH8CUlt4feAPwAeBv54\nmHO2AtPKtqcC2wYpW7HWAVwNLI2I/eU7JX0J6AQ+O9gXj4g7IqI7Iro7OzuHCdWy1NGRTODoGolZ\na6pq+G9EHCJ5uv1rI7j2KmC2pBnAj0mSxa8NLCRpDjAJeKLCNRYBnx9Q/nrgPwMfTeOyJtDVBY8/\nnncUZpaFaufami1piaR1kjaXXkOdExEHgBtJmqXWA4sjYq2kWyVdUlZ0EVBMO8/Lv+Z0khrNowMu\n/ffAFOCJdGjwF6u5B8tXoQAvvQSvv553JGZWb1XVSIBvAl8CbgMuAH6Tyn0gh4mI5cDyAfu+OGD7\ny4Ocu4UKnfMRUW3M1kBKHe5r18Iv/3K+sZhZfVXbR3J0RDwMKCJeTH/5X5hdWNZqyhOJmbWWav+6\n35tOIb9J0o0kfR4nZReWtZrTT4djj3WHu1krqrZG8t+AY4DPAL8E/Bfg2qyCstZzxBEwb54TiVkr\nGrZGkj58eHVE/BGwi6R/xGzECgVYvnz4cmbWXIatkaQPCf5S+ZPtZqNRKMArr8Brr+UdiZnVU7V9\nJM8C35Z0N/BmaWdE3JtJVNaSyjvcP/zhfGMxy8qePfCZz8COHXlHkvibv4FTK05OVT/VJpITgJ9y\n+EitIJ2t16wa5XNuOZFYq1q6FL7xDTjzTDiyARa5eOut7L9GtU+2u1/EanbyyTBxojvcrbX19CQ1\ngPXrk0Em7aDaFRK/SYUJFyPit+oekbUsyYtcWWvbsQPuvx9uuql9kghU37T13bLPE4BPMfgEjGaD\nKhSgWEzWJ/HwDWs1990H+/fDNdfkHcnYqrZp657ybUl3Ad/PJCJraYUC7NwJ27fDKafkHY1ZffX0\nwIwZ8J735B3J2Bpt5Ws2cFo9A7H24EWurFX198P3vw8LF7Zfbbva2X/fkPR66QV8h2SNErMR6epK\n3p1IrNXcey8cPNh+zVpQfdPWO7MOxNrD5MkwZYoTibWeYhHmzoVzzsk7krFXbY3kU5KOL9ueKOmy\n7MKyVuaRW9Zqtm2DRx9NaiPt1qwF1feRfCkiflbaiIidJOuTmI1YoQDr1sEhr29pLWLJkmQkYjs2\na0H1iaRSOS8wZaNSKMCbb8KLL+YdiVl9FItJk9ZZZ+UdST6qTSS9kv5a0hmSZkq6DXg6y8CsdbnD\n3VrJiy/CE08ko7XaVbWJ5CZgH9ADLAb2AH+QVVDW2pxIrJUsXpy8t2uzFlQ/autN4JaMY7E2cdxx\ncNppXnbXWkOxmDyAOHNm3pHkp9pRWw9Jmli2PUnSg9mFZa3OI7esFWzaBM88097NWlB909bkdKQW\nABGxgyrWbJe0QNIGSX2S3lajkXSbpNXpa6Oknen+C8r2r5a0tzTcWNIMSU9J2iSpR9L4Ku/BGkih\nkMyOeuBA3pGYjV5PT/J+1VX5xpG3ahPJIUk/nxJF0nQqzAZcLl2i93bg48A8YJGkeeVlIuLmiDgv\nIs4Dvkq6vklEPFK2/0JgN7AiPe2vgNsiYjawA/jtKu/BGkhXF+zbB319eUdiNno9PfDBD8K0aXlH\nkq9qE8kXgMcl/YukfwEeBT4/zDnzgb6I2BwR+4AicOkQ5RcBd1XYfyVwf0TsTpf7vRBYkh77J8AP\nRjYhz7llzW7NmuTV7s1aUGUiiYgHgG5gA8nIrf9OMnJrKKcCL5dtb033vY2k04EZwMoKhxfyiwRz\nIrAzIkoNIoNe0xrbWWclTwC7w92aVU9PsubIlVfmHUn+ql3Y6nrgD4GpwGrgfcATHL707ttOq7Bv\nsOawhcCSiDg44OueDJwNlDr2q76mpBuAGwBOO80TFTeao4+GWbNcI7HmFJEkkgsuSOaOa3fVNm39\nIfAe4MWIuAA4H+gf5pytQHnL4VQGXwyrvNZR7mpgaUTsT7dfAyZKKiXAQa8ZEXdERHdEdHd2dg4T\nquXBI7esWT37bDJiq52fHSlXbSLZGxF7ASQdFRHPA3OGOWcVMDsdZTWeJFksG1hI0hxgEkkNZ6DD\n+k0iIoBHSPpNAK4Fvl3lPViD6epK/jPu3Zt3JGYj09MDHR1w+eV5R9IYqk0kW9PnSO4DHpL0bYZZ\najftx7iRpFlqPbA4ItZKulXSJWVFFwHFNEn8XDoybBpJx365PwE+K6mPpM/kf1d5D9ZgCoVk/YYN\nG/KOxKx6pWatiy+GE0/MO5rGUO2T7Z9KP35Z0iPA8cADVZy3HFg+YN8XB2x/eZBzt1ChIz0iNpOM\nCLMmVxq5tXYtnHtuvrGYVeupp5L5tW69Ne9IGseIZ/CNiIE1BLNRmT0bjjzS/STWXIpFOOoouHSo\nhxnazGjXbDer2fjxMGeOE4k1j4MHk0kaP/5xOP744cu3CycSy1VXlxOJNY/HH4ft2/0Q4kBOJJar\nQgF+9CPYtSvvSMyGVyzCMcfAJz+ZdySNxYnEclXqcF+/Pt84zIZz4ECypO6v/ioce2ze0TQWJxLL\nlefcsmaxciW89pqbtSpxIrFczZiRTJfiRGKNrlhMFmVbsCDvSBqPE4nlaty4ZAJHJxJrZG+9BUuX\nwmWXwYQJeUfTeJxILHeec8sa3YoVsHOn59YajBOJ5a5QgG3bYMeOvCMxq6ynB044AS66KO9IGpMT\nieWufKoUs0azezd8+9twxRXJQ7T2dk4klruuruTdzVvWiJYvT55zcrPW4JxILHfTpsE73+lEYo2p\npydZvOojH8k7ksblRGK5k9zhbo3pjTfgu99NltMdNy7vaBqXE4k1hFIiicEWYzbLwXe+kyy85ocQ\nh+ZEYg2hUICf/hRefTXvSMx+oViEqVPh/e/PO5LG5kRiDcEd7tZoduyABx6Aq6+GI/ybckj+9lhD\n8Jxb1mjuuw/273ezVjWcSKwhnHQSTJ7sRGKNo1iEmTOhuzvvSBqfE4k1hNLILT+UaI2gvx8efjh5\ndkTKO5rG50RiDcMjt6xR3HNPsqyum7Wqk2kikbRA0gZJfZJuqXD8Nkmr09dGSTvLjp0maYWk9ZLW\nSZqe7v+opGfScx6XNCvLe7Cx09WVjNt/+eW8I7F219MDc+fC2WfnHUlzyCyRSBoH3A58HJgHLJI0\nr7xMRNwcEedFxHnAV4F7yw7/M/CViDgLmA+UBoZ+Dfj19Jz/A/xpVvdgY8sd7tYItm2DRx9NaiNu\n1qpOljWS+UBfRGyOiH1AEbh0iPKLgLsA0oTTEREPAUTErojYnZYL4Lj08/HAtiyCt7HnIcDWCO6+\nO2le9dxa1evI8NqnAuWNFFuB91YqKOl0YAawMt11JrBT0r3p/u8Dt0TEQeB6YLmkPcDrwPuyCd/G\n2qRJcOqp7nC3fPX0wLnnJk1bVp0saySVKoWDdaMuBJakiQKSBPch4HPAe4CZwHXpsZuBT0TEVOCb\nwF9X/OLSDZJ6JfX29/eP7g5szHnOLcvTiy/CE0+4k32kskwkW4FpZdtTGbwZaiFps1bZuc+mzWIH\ngPuAd0vqBM6NiKfScj1AxckLIuKOiOiOiO7Ozs5a7sPGUFcXrFuXjJgxG2uLFyfvV1+dbxzNJstE\nsgqYLWmGpPEkyWLZwEKS5gCTgCcGnDspTRwAFwLrgB3A8ZLOTPd/DFifUfyWg0IhmSRv8+a8I7F2\nVCzC/PnJg4hWvcwSSVqTuBF4kOSX/eKIWCvpVkmXlBVdBBQjfvH0QNrE9TngYUnPkTSTfT295qeB\neyT9EPgN4I+yugcbex65ZXnZtAmeecad7KORZWc7EbEcWD5g3xcHbH95kHMfAs6psH8psLR+UVoj\nmZcOEF+7Fj71qXxjsfbS05O8u1lr5PxkuzWUY49NmhVcI7GxVizCBz+YTBtvI+NEYg2nq8uJxMbW\nmjVJLdijtUbHicQaTqEAGzbAvn15R2LtoqcnWXPkyivzjqQ5OZFYwykU4MAB2Lgx70isHUQkzVoX\nXABTpuQdTXNyIrGGUxq55SfcbSw8+yz09blZqxZOJNZw5syBcePcT2Jjo1iEjg64/PK8I2leTiTW\ncI46CmbPdiKx7EUk/SMXXwwnnJB3NM3LicQakufcsrHw5JPw0ktu1qqVE4k1pEIBXngB9uzJOxJr\nZcViUgO+dKgFLmxYTiTWkAqFpNlhvWdSs4wcPJisPfKJT8Bxxw1f3gbnRGINyXNuWdb+/d9h+3bP\nrVUPTiTWkM44A8aPdyKx7PT0wDHHwCc/mXckzc+JxBpSRwecdZYTiWXjwAFYsgQuuSSZ381q40Ri\nDatQ8EOJlo2VK+G119ysVS9OJNawCoVkaObrr+cdibWaYjHpYF+wIO9IWoMTiTUsT5ViWXjrLbj3\nXrjsMpgwIe9oWoMTiTWsrq7k3f0kVk8rVsDPfuaHEOvJicQa1umnJx2hTiRWT8ViMh3KRRflHUnr\ncCKxhnXEEUmtxE1bVi+7d8OyZXDFFXDkkXlH0zqcSKyhec4tq6fly2HXLjdr1ZsTiTW0QgFeeQX6\n+/OOxFpBsZgsXvXhD+cdSWtxIrGGVupwd/OW1eqNN+B734OrrkrWu7H6yTSRSFogaYOkPkm3VDh+\nm6TV6WujpJ1lx06TtELSeknrJE1P90vSn6fl10v6TJb3YPnynFtWL8uWwd69fggxCx1ZXVjSOOB2\n4GPAVmCVpGURsa5UJiJuLit/E3B+2SX+GfjziHhI0juAQ+n+64BpwNyIOCTppKzuwfJ38skwaZJr\nJFa7nh6YOhXe//68I2k9WdZI5gN9EbE5IvYBRWCoWf8XAXcBSJoHdETEQwARsSsidqflfg+4NSIO\npcdezeoGLH+SO9ytdjt2wAMPJLWRI9ygX3dZfktPBV4u296a7nsbSacDM4CV6a4zgZ2S7pX0rKSv\npDUcgDOAayT1Srpf0uxBrnlDWqa33z21Ta2rK0kkEXlHYs1q6VLYv9/NWlnJMpGowr7BfhUsBJZE\nxMF0uwP4EPA54D3ATJImLYCjgL0R0Q18Hbiz0gUj4o6I6I6I7s7OztHdgTWEQgF27oRt2/KOxJpV\nTw/MnAnd3XlH0pqyTCRbSfoySqYCg/0qWEjarFV27rNps9gB4D7g3WXH7kk/LwXOqVvE1pDc4W61\n6O+Hhx9OaiOq9Oet1SzLRLIKmC1phqTxJMli2cBCkuYAk4AnBpw7SVKpKnEhUOqkvy/dBvgwsDGD\n2K2BeAiw1eKee5Jldf0QYnYySyRpTeJG4EFgPbA4ItZKulXSJWVFFwHFiF+0gKdNXJ8DHpb0HEkz\n2dfTw38JXJHu/wvg+qzuwRrD5Mnwrne5RmKjUywmi6SdfXbekbSuzIb/AkTEcmD5gH1fHLD95UHO\nfYgKzVYRsRP4lfpFac2g1OFuNhLbtsFjj8GXvuRmrSx5IJw1hdJqiYcODV/WrOTuu5PRfh6tlS0n\nEmsKhUIyc+uWLXlHYs2kWIRzz4W5c/OOpLU5kVhT8GqJNlJbtsCTT7qTfSw4kVhTmDcveXc/iVVr\n8eLk3c1a2XMisaZw3HFw2mlOJFa9nh6YPx9mzMg7ktbnRGJNw3NuWbU2boRnnnGz1lhxIrGmUSjA\n88/DgQN5R2KNrqcneb/qqnzjaBdOJNY0CgXYtw/6+vKOxBpdTw986EPJtPGWPScSaxqec8uqsWZN\nMrrPzVpjx4nEmsbcuclaEk4kNpSenuTfyRVX5B1J+3AisaZx9NFwxhlOJDa4iOQhxAsvhClT8o6m\nfTiRWFPxyC0byjPPJH1ofnZkbDmRWFMpFJJfFHv35h2JNaKeHujogMsvzzuS9pLp7L9m9VYoJGtL\nnHMOHHlk3tFYo9myBS6+GE44Ie9I2osTiTWViy+Ga6+FN9/MOxJrRIUCfPazeUfRfpxIrKlMnAj/\n+I95R2Fm5dxHYmZmNXEiMTOzmjiRmJlZTZxIzMysJk4kZmZWEycSMzOriROJmZnVxInEzMxqoojI\nO4bMSeoHXhzl6ZOB1+oYTp5a5V5a5T7A99KoWuVear2P0yOic7hCbZFIaiGpNyK6846jHlrlXlrl\nPsD30qha5V7G6j7ctGVmZjVxIjEzs5o4kQzvjrwDqKNWuZdWuQ/wvTSqVrmXMbkP95GYmVlNXCMx\nM7OaOJEMIOkrkp6X9B+SlkqaOEi5BZI2SOqTdMtYx1kNSVdJWivpkKRBR25I2iLpOUmrJfWOZYzV\nGMF9NMPP5ARJD0nalL5PGqTcwfTnsVrSsrGOcyjDfZ8lHSWpJz3+lKTpYx/l8Kq4j+sk9Zf9HK7P\nI85qSLpT0quS1gxyXJL+V3qv/yHp3XUNICL8KnsBFwMd6ee/Av6qQplxwAvATGA88ENgXt6xV4jz\nLGAO8G9A9xDltgCT8463lvtoop/J/wRuST/fUunfV3psV96xjvb7DPw+8Pfp54VAT95xj/I+rgP+\nNu9Yq7yf/wS8G1gzyPFPAPcDAt4HPFXPr+8ayQARsSIiDqSbTwJTKxSbD/RFxOaI2AcUgUvHKsZq\nRcT6iNiQdxy1qvI+muJnQhLTP6Wf/wm4LMdYRqOa73P5PS4BPipJYxhjNZrl30tVIuIx4P8NUeRS\n4J8j8SQwUdLJ9fr6TiRD+y2SLD7QqcDLZdtb033NKoAVkp6WdEPewYxSs/xMpkTEdoD0/aRByk2Q\n1CvpSUmNlGyq+T7/vEz6R9nPgBPHJLrqVfvv5Yq0KWiJpGljE1omMv3/0ZZrtkv6PvCuCoe+EBHf\nTst8ATgAfKvSJSrsy2X4WzX3UoUPRMQ2SScBD0l6Pv0LZ8zU4T6a4mcygsuclv5MZgIrJT0XES/U\nJ8KaVPN9bpifxRCqifE7wF0R8Zak3yWpZV2YeWTZyPRn0paJJCIuGuq4pGuBTwIfjbSBcYCtQPlf\nJ1OBbfWLsHrD3UuV19iWvr8qaSlJtX9ME0kd7qMpfiaSXpF0ckRsT5sWXh3kGqWfyWZJ/wacT9Km\nn7dqvs+lMlsldQDHM3SzSx6GvY+I+GnZ5tdJ+kybVab/P9y0NYCkBcCfAJdExO5Biq0CZkuaIWk8\nSYdiQ42sqZakYyW9s/SZZLBQdK4VAAADEUlEQVRBxZEfDa5ZfibLgGvTz9cCb6ttSZok6aj082Tg\nA8C6MYtwaNV8n8vv8Upg5SB/kOVp2PsY0IdwCbB+DOOrt2XAf01Hb70P+FmpibUu8h5t0GgvoI+k\nLXF1+iqNPjkFWF5W7hPARpK/Er+Qd9yD3MunSP4SeQt4BXhw4L2QjFr5Yfpa24j3Us19NNHP5ETg\nYWBT+n5Cur8b+Eb6+f3Ac+nP5Dngt/OOe8A9vO37DNxK8scXwATg7vT/0g+AmXnHPMr7+Iv0/8QP\ngUeAuXnHPMS93AVsB/an/1d+G/hd4HfT4wJuT+/1OYYYxTmal59sNzOzmrhpy8zMauJEYmZmNXEi\nMTOzmjiRmJlZTZxIzMysJk4kZnUgaVeN5y9Jn2JH0jsk/YOkF9JZjx+T9F5J49PPbfkgsTUuJxKz\nnEnqAsZFxOZ01zdIngSfHRFdJLPQTo5kcsGHgWtyCdRsEE4kZnWUPjn8FUlr0jVerkn3HyHp79Ia\nxnclLZd0ZXrar5M+4S7pDOC9wJ9GxCFIpkmJiO+lZe9Ly5s1DFeRzerrcuA84FxgMrBK0mMk05xM\nB84mmfF3PXBnes4HSJ5MBugCVkfEwUGuvwZ4TyaRm42SayRm9fVBkhljD0bEK8CjJL/4PwjcHRGH\nIuInJFNulJwM9Fdz8TTB7CvNj2bWCJxIzOprsAWchlrYaQ/J/FSQzO10rqSh/m8eBewdRWxmmXAi\nMauvx4BrJI2T1EmyBOoPgMdJFkk6QtIU4CNl56wHZgFEsuZIL/BnpVUFJc2WdGn6+USgPyL2j9UN\nmQ3HicSsvpYC/0EyY+xK4I/Tpqx7SGZlXQP8A/AUycqBAN/j8MRyPcnCWH2SniNZC6O0dsQFwPJs\nb8FsZDz7r9kYkfSOiNiV1ip+QLIy5U8kHU3SZ/KBITrZS9e4F/h8DL+GvdmY8agts7HzXUkTgfHA\n/0hrKkTEHklfIllD+6XBTk4XYLrPScQajWskZmZWE/eRmJlZTZxIzMysJk4kZmZWEycSMzOriROJ\nmZnVxInEzMxq8v8B7zlu2lY2hUgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x150d5860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-2, 1, 10)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份  \n",
    "#penalty_s = ['l1','l2']\n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "#    for j, penalty in enumerate(penalty_s):\n",
    "    tmp = fit_grid_point_Linear(oneC, X_train, y_train, X_test, y_test)\n",
    "    accuracy_s.append(tmp)\n",
    "\n",
    "x_axis = np.log10(C_s)\n",
    "#for j, penalty in enumerate(penalty_s):\n",
    "plt.plot(x_axis, np.array(accuracy_s), 'b-')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuracy' )\n",
    "plt.savefig('SVM_Otto.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RBF核SVM正则参数调优\n",
    "\n",
    "RBF核是SVM最常用的核函数。\n",
    "RBF核SVM 的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和核函数的宽度gamma\n",
    "C越小，决策边界越平滑； \n",
    "gamma越小，决策边界越平滑。\n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fit_grid_point_RBF(C, gamma, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC3 =  SVC( C = C, kernel='rbf', gamma = gamma)\n",
    "    SVC3 = SVC3.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC3.score(X_val, y_val)\n",
    "    \n",
    "    print(\"accuracy: {}\".format(accuracy),\"C=%f\"%(C),\"gamma=%f\"%(gamma))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.6601307189542484 C=0.010000 gamma=0.010000\n",
      "accuracy: 0.6601307189542484 C=0.010000 gamma=0.100000\n",
      "accuracy: 0.6601307189542484 C=0.010000 gamma=1.000000\n",
      "accuracy: 0.6601307189542484 C=0.010000 gamma=10.000000\n",
      "accuracy: 0.6601307189542484 C=0.010000 gamma=100.000000\n",
      "accuracy: 0.6601307189542484 C=0.021544 gamma=0.010000\n",
      "accuracy: 0.6601307189542484 C=0.021544 gamma=0.100000\n",
      "accuracy: 0.6601307189542484 C=0.021544 gamma=1.000000\n",
      "accuracy: 0.6601307189542484 C=0.021544 gamma=10.000000\n",
      "accuracy: 0.6601307189542484 C=0.021544 gamma=100.000000\n",
      "accuracy: 0.6601307189542484 C=0.046416 gamma=0.010000\n",
      "accuracy: 0.6666666666666666 C=0.046416 gamma=0.100000\n",
      "accuracy: 0.6601307189542484 C=0.046416 gamma=1.000000\n",
      "accuracy: 0.6601307189542484 C=0.046416 gamma=10.000000\n",
      "accuracy: 0.6601307189542484 C=0.046416 gamma=100.000000\n",
      "accuracy: 0.6601307189542484 C=0.100000 gamma=0.010000\n",
      "accuracy: 0.7581699346405228 C=0.100000 gamma=0.100000\n",
      "accuracy: 0.6601307189542484 C=0.100000 gamma=1.000000\n",
      "accuracy: 0.6601307189542484 C=0.100000 gamma=10.000000\n",
      "accuracy: 0.6601307189542484 C=0.100000 gamma=100.000000\n",
      "accuracy: 0.7581699346405228 C=0.215443 gamma=0.010000\n",
      "accuracy: 0.7712418300653595 C=0.215443 gamma=0.100000\n",
      "accuracy: 0.6601307189542484 C=0.215443 gamma=1.000000\n",
      "accuracy: 0.6601307189542484 C=0.215443 gamma=10.000000\n",
      "accuracy: 0.6601307189542484 C=0.215443 gamma=100.000000\n",
      "accuracy: 0.7712418300653595 C=0.464159 gamma=0.010000\n",
      "accuracy: 0.7712418300653595 C=0.464159 gamma=0.100000\n",
      "accuracy: 0.6601307189542484 C=0.464159 gamma=1.000000\n",
      "accuracy: 0.6601307189542484 C=0.464159 gamma=10.000000\n",
      "accuracy: 0.6601307189542484 C=0.464159 gamma=100.000000\n",
      "accuracy: 0.7647058823529411 C=1.000000 gamma=0.010000\n",
      "accuracy: 0.7581699346405228 C=1.000000 gamma=0.100000\n",
      "accuracy: 0.6797385620915033 C=1.000000 gamma=1.000000\n",
      "accuracy: 0.6601307189542484 C=1.000000 gamma=10.000000\n",
      "accuracy: 0.6601307189542484 C=1.000000 gamma=100.000000\n",
      "accuracy: 0.7712418300653595 C=2.154435 gamma=0.010000\n",
      "accuracy: 0.7581699346405228 C=2.154435 gamma=0.100000\n",
      "accuracy: 0.673202614379085 C=2.154435 gamma=1.000000\n",
      "accuracy: 0.6601307189542484 C=2.154435 gamma=10.000000\n",
      "accuracy: 0.6601307189542484 C=2.154435 gamma=100.000000\n",
      "accuracy: 0.7777777777777778 C=4.641589 gamma=0.010000\n",
      "accuracy: 0.7777777777777778 C=4.641589 gamma=0.100000\n",
      "accuracy: 0.6797385620915033 C=4.641589 gamma=1.000000\n",
      "accuracy: 0.6601307189542484 C=4.641589 gamma=10.000000\n",
      "accuracy: 0.6601307189542484 C=4.641589 gamma=100.000000\n",
      "accuracy: 0.7843137254901961 C=10.000000 gamma=0.010000\n",
      "accuracy: 0.7712418300653595 C=10.000000 gamma=0.100000\n",
      "accuracy: 0.6797385620915033 C=10.000000 gamma=1.000000\n",
      "accuracy: 0.6601307189542484 C=10.000000 gamma=10.000000\n",
      "accuracy: 0.6601307189542484 C=10.000000 gamma=100.000000\n"
     ]
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-2, 1, 10)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "gamma_s = np.logspace(-2, 2, 5)    \n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "    for j, gamma in enumerate(gamma_s):\n",
    "        tmp = fit_grid_point_RBF(oneC, gamma, X_train, y_train, X_test, y_test)\n",
    "        accuracy_s.append(tmp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上述部分运行结果来看，gamma参数设置不合适（gamma越大，对应RBF核的sigma越小，决策边界更复杂，可能发生了过拟合）\n",
    "所以调小gamma值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.6601307189542484 C=0.010000 gamma=0.010000\n",
      "accuracy: 0.6601307189542484 C=0.010000 gamma=0.017783\n",
      "accuracy: 0.6601307189542484 C=0.010000 gamma=0.031623\n",
      "accuracy: 0.6601307189542484 C=0.010000 gamma=0.056234\n",
      "accuracy: 0.6601307189542484 C=0.010000 gamma=0.100000\n",
      "accuracy: 0.6601307189542484 C=0.021544 gamma=0.010000\n",
      "accuracy: 0.6601307189542484 C=0.021544 gamma=0.017783\n",
      "accuracy: 0.6601307189542484 C=0.021544 gamma=0.031623\n",
      "accuracy: 0.6601307189542484 C=0.021544 gamma=0.056234\n",
      "accuracy: 0.6601307189542484 C=0.021544 gamma=0.100000\n",
      "accuracy: 0.6601307189542484 C=0.046416 gamma=0.010000\n",
      "accuracy: 0.6601307189542484 C=0.046416 gamma=0.017783\n",
      "accuracy: 0.6601307189542484 C=0.046416 gamma=0.031623\n",
      "accuracy: 0.673202614379085 C=0.046416 gamma=0.056234\n",
      "accuracy: 0.6666666666666666 C=0.046416 gamma=0.100000\n",
      "accuracy: 0.6601307189542484 C=0.100000 gamma=0.010000\n",
      "accuracy: 0.738562091503268 C=0.100000 gamma=0.017783\n",
      "accuracy: 0.7647058823529411 C=0.100000 gamma=0.031623\n",
      "accuracy: 0.7712418300653595 C=0.100000 gamma=0.056234\n",
      "accuracy: 0.7581699346405228 C=0.100000 gamma=0.100000\n",
      "accuracy: 0.7581699346405228 C=0.215443 gamma=0.010000\n",
      "accuracy: 0.7777777777777778 C=0.215443 gamma=0.017783\n",
      "accuracy: 0.7712418300653595 C=0.215443 gamma=0.031623\n",
      "accuracy: 0.7712418300653595 C=0.215443 gamma=0.056234\n",
      "accuracy: 0.7712418300653595 C=0.215443 gamma=0.100000\n",
      "accuracy: 0.7712418300653595 C=0.464159 gamma=0.010000\n",
      "accuracy: 0.7777777777777778 C=0.464159 gamma=0.017783\n",
      "accuracy: 0.7843137254901961 C=0.464159 gamma=0.031623\n",
      "accuracy: 0.7843137254901961 C=0.464159 gamma=0.056234\n",
      "accuracy: 0.7712418300653595 C=0.464159 gamma=0.100000\n",
      "accuracy: 0.7647058823529411 C=1.000000 gamma=0.010000\n",
      "accuracy: 0.7843137254901961 C=1.000000 gamma=0.017783\n",
      "accuracy: 0.7843137254901961 C=1.000000 gamma=0.031623\n",
      "accuracy: 0.7777777777777778 C=1.000000 gamma=0.056234\n",
      "accuracy: 0.7581699346405228 C=1.000000 gamma=0.100000\n",
      "accuracy: 0.7712418300653595 C=2.154435 gamma=0.010000\n",
      "accuracy: 0.7777777777777778 C=2.154435 gamma=0.017783\n",
      "accuracy: 0.7777777777777778 C=2.154435 gamma=0.031623\n",
      "accuracy: 0.7712418300653595 C=2.154435 gamma=0.056234\n",
      "accuracy: 0.7581699346405228 C=2.154435 gamma=0.100000\n",
      "accuracy: 0.7777777777777778 C=4.641589 gamma=0.010000\n",
      "accuracy: 0.7777777777777778 C=4.641589 gamma=0.017783\n",
      "accuracy: 0.7712418300653595 C=4.641589 gamma=0.031623\n",
      "accuracy: 0.7843137254901961 C=4.641589 gamma=0.056234\n",
      "accuracy: 0.7777777777777778 C=4.641589 gamma=0.100000\n",
      "accuracy: 0.7843137254901961 C=10.000000 gamma=0.010000\n",
      "accuracy: 0.7647058823529411 C=10.000000 gamma=0.017783\n",
      "accuracy: 0.7908496732026143 C=10.000000 gamma=0.031623\n",
      "accuracy: 0.7712418300653595 C=10.000000 gamma=0.056234\n",
      "accuracy: 0.7712418300653595 C=10.000000 gamma=0.100000\n"
     ]
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-2, 1, 10)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "gamma_s = np.logspace(-2, -1, 5)  \n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "    for j, gamma in enumerate(gamma_s):\n",
    "        tmp = fit_grid_point_RBF(oneC, gamma, X_train, y_train, X_test, y_test)\n",
    "        accuracy_s.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xl4lNXZ+PHvmcm+7yEQsoeEELKx\nb2FxQ3GvFvWnr1qXvq7Vtra2Um1tsW7VWrW+te7autaqVWRfEgSRbBACZAGykn1fZpJZzu+PCTFA\nQiYhM0no+VwXl8zMeZ7nTAxzz3POfe4jpJQoiqIoyploxroDiqIoyvingoWiKIoyJBUsFEVRlCGp\nYKEoiqIMSQULRVEUZUgqWCiKoihDUsFCURRFGZIKFoqiKMqQVLBQFEVRhuQw1h0YLQEBATIiImKs\nu6EoijKhZGdnN0gpA4dqd84Ei4iICLKyssa6G4qiKBOKEKLMmnZqGEpRFEUZkgoWiqIoypBUsFAU\nRVGGpIKFoiiKMiQVLBRFUZQhqWChKIqiDEkFC0VRFGVIKlgoiqJMYNvKt/F5yec2v44KFoqiKBPU\n+4ff54HtD/BJ0SeYzCabXuucWcGtKIry38Iszfw5+8+8WfAmy0KX8VT6U2g1WpteUwULRVGUCaTb\n1M2anWtYX7qe1XGr+dXcX9k8UIAKFoqiKBNGa3cr92+9n5y6HH4666fcMuMWhBB2ubYKFoqiKBNA\nZXsld22+i6qOKp5Of5qLIy+26/VVsFAURRnnChoKuGfLPfSYe3j1gleZPWm23fugsqEURVHGsR0V\nO7h1w604a5157+L3xiRQgAoWiqIo49ZHhR9x/7b7ifCK4B+r/kGUT9SY9UUNQymKoowzZmnmLzl/\n4fUDr7NkyhKeXfosbo5uY9onFSwUZZxrrD7G7hd/Q4uDgaqkSbRP8rR/J0w90FaNtquBJL8EliTd\nSlBwov37MU5IKWnftImeY6X4rv4hWh+fUTt3j6mH33zzG9YdW8c1067hkXmP4KA5/aPabJbsr2pl\n6+E6HDSC+8+LHbU+DEQFC0UZpww9erb95WH83ttIpF6iAWZ9tJ96fwcKprlwcJorJZHOmBxslDpp\nMoBRBwY9mLoB0AkNn+or4fhGppu1LPGJIz32KhLjr0br4GSbfowz+sIiap94gq49ewBoevNNAh/4\nCT7XXovQnt16h9buVh7Y9gBZtVn8JO0n3JZ420mpsa06A5nF9Ww7XM+OojoaOnrQCDhvevBZXdca\nQkpp84vYw+zZs6Xag1s5V2R99Sbtz7zApJpuyuK8ifntHwkLjKVjxw46duyga893yO5uhJsb7gsW\n4LE0HY+lS3EMPosPjZ4uOJYBxRugeBO0VlienzQTYi+CaRchQ1IpOraRzEMfkdF0gH10YxYCX7Nk\nkVMg6aFLWZjyI7y9w0bnBzGOmFpaqH/xJZo/+ACthweBD/wE1+Rkap98iq7vvsM5Pp5Jj/watzlz\nRnT+4x3HuXvz3ZS1l/H7Rb/n0qhLkVJSUtfB1sN1bD1cR1ZZMyazxNvVkaXTAlkRH8TSaYH4uo88\nUAshsqWUQ86aq2ChKONIRVE2+x/7GVG5tTT6ahH3/4gFqx9Aozk5F8Ws09G5Z09f8DAerwbAOT4e\nj6VL8Vi6FNfkpKG/6TaXQtFGS4A4lmm5g3B0h+jlEHsBxF4IXpMHPbyl+Ri79r9FRmUGO3vqadUI\ntFKSjDPp/kmkT19NTNSFCM3EzaWRJhMtH39M/Z9fwNTWhu91qwm47z4cfH0tr0tJ+4YN1D71NMbq\narwuuYSgh36OY0iI1dc42HiQe7bcQ7exm6fTn8PQEcW2QkuAqGzWARA/yZMV8UGsiA8iZaoPDtrR\n+ZmOi2AhhFgJvABogdeklE+e8vrzwPLeh25AkJTSp/e1p4FVWDK2NgE/kWforAoWykTW2d7Ejicf\nYMrne5ECqq9awNJfPIebx9Bj4VJKuouL6dixg84dGXTl5oLJhNbHB/clSyzBY/Eiy7i6yQDlu6F4\noyVINBRaTuIX1Xv3cCGELwIH52G/B5Oxh/xDH5NR/BmZrUUc1pgBCDFJlriFkh5xIXOTbsHVzW/Y\n5x4rXVlZ1Kx9gu5Dh3CbM4fgNY/gEhc3YFuzTkfja6/T+NproNEQcOcd+P3oR2icz/yzzKzM5Kfb\nf4aj8CDccD+5R5zRG8y4OmpZFOPP8vgglscFMdnH1RZvceyDhRBCCxQBFwCVwF7geinlwUHa3wek\nSil/JIRYCDwDpPe+vBP4lZRy+2DXU8FCmYjMZjM733sG7cvv4tdq4uicyaT+9nkmRyeN+Jym1lY6\nv/nGcteRkYmpuRmEwHWqOx4BjXgEteLsKxCRi/qGl/CPHsV3ZVFbu5/M/W+RUb2Hb02t6DQCZ7Nk\njsaD9ODZpM+8iSlT5o36dUeDobqaumeepW3dOhxCQgj+xUN4rlxpVWmNnsoq6p5+mvaNG3EMDSXo\nl7/A8/zzTzrWaDKTXdbMq3nv813b3zF1B6OruJWpXpNYER/E8vgg5kX64eJo+5pP4yFYLAB+K6W8\nqPfxrwCklH8cpP0u4DEp5abeY18CFgMCyABuklIeGux6KlgoE83h7zZQ9rvfEHakneoQF3we/hlp\nF9149ic2m6E6F4o3IQ+vR19wgI7jLnTUeqBvsHxgOUwKxmPpMjyWLsV9/jw0brZNy+zpbicr/10y\nj35NRkcp5b2fgdEmwRKvaNKjLyUl8Xocxzg91NzdTdObb9Lwt1fBbMb/ttvwv+N2NK7D/1bf+e23\n1K5dS3dxCe4LF+D8wM/5xujF1sN17Ciqo9vza5wDtuJhnsFNUWu4cHoE0YHudqv1dMJ4CBbXACul\nlLf3Pr4JmCelvHeAtuHAt0ColNLU+9yzwO1YgsVLUspHznQ9FSyUiaKppoxdj99H5LZiOl0EbTev\nYtk9a3FwPItsIn0rHNlmGV4q3gSddYCA0NnfDy9NSsJQV09nZgYdOzLo/OYbzF1dCCcn3ObOtQxX\nLVuK09Spo/ZeB1NauoOMg/8goz6XbKnDKASeZskCR1/SQxaxOPlW/AMGHu6xBSklHVu2UPvkUxgq\nK/G88EKCfvELnEKnnNU5CyqaKP77O4R/8R5OPXr+E7mIdbMvxm3aRo6bdnJZ5JX8bvGjOGocR/Hd\nDM94CBbXAhedEizmSinvG6DtL7EEivt6H8dgmetY3dtkE/BLKWXGKcfdCdwJEBYWNqusrMwm70VR\nRoPR0MO2l36NzzvrcNNLjq2IY9Fjf8E3aASZQ1JCQxEUbbAEiPLdYDaCizfEnG8JEDHng7v/4Kfo\n6aErO5uO7ZZJ8p7SUgCcoqLwSE/HY9lS3NLSEE62TYnt7Kjh231vkVG+hUxdNfVagZCSGdKRdN/p\npMf9gOnTrkCjtU2mf3dJCbVPPEHnrt04x8YQ/MgjuM+fP6JzdXQb2VncwLbDdWwrrKOuvRshYL6/\nllsPbSB01wZ0blreTTcz/X/u5c6U/7X7ncSpxkOwsHoYSgiRC9wjpdzV+/ghwEVK+fvex48Ceinl\n04NdT91ZKONZ9vp3aX3qT4RUd1Me40X4o48TP/ei4Z3EoIfSnZbMpaIN0NL75ShohiVzadpFEDoX\nRvih2lNWRseODEtq7nffIQ0GNO7uuC9aZLnrSF+CQ2DgiM5tLWk2c6j4CzIP/4uMpgLyRQ9SCAJM\nksUuk0gPW86C5B/h4Wl9ptFgTG1tNLz8Mk3v/QONuzuB992H7/XXIRyG9/M7Wm9Jbd1WWMd3x5ow\nmCSeLg6kTwtkeVwQy+ICCfBwpqazhsffvpULPi0jvlLikpBA8JpHcEtLO+v3cjbGQ7BwwDLBfR5Q\nhWWC+wYpZcEp7eKADUDkiWwnIcRq4A5gJZZhqPXAn6WU/xnseipYKDax+2UoWj/iw4+36cndWkPU\nISNNXmA+z4dFcX5oxDDTHk1GOJ5rWSTn4ApRSy1prbEXgs/oDxuZOzvp/PZby11HRgbG2loAXBIT\n++46XBITbZ4S29RUwjf73iSzaic7DY20awQOUpImnUjXeLAEVyJxRGD9t3MpJa3726nb3oipy4RP\nqheB6f44uFk3mWyW0KYzcKA2hiOtCWwXAex1cCdkkkff5PSscF8c+6W2FjYVcvfmu+kydvH8sueY\nntNE3TPPYKytxeuyywj6+c/Obo3MWRjzYNHbiUuAP2NJnX1DSrlWCPE4kCWl/KK3zW+x3EU83O84\nLfBXLNlQElgvpfzpma6lgoUy6iqz4bUVEBAHw0z37DKY2L63gcnf6hFA1TwXls71x91xpEMpAiYl\nWoaXIhaDo8sIzzN8Ukq6Cwv7hqt0+/aB2YzW3x+PJUvwWLYU94UL0Xp52bQfRoOefQc/JKPkCzLa\nj1EiDACESi3p0oUl0oU5OON8hsDRVaWndnMz+toeXEOdmXSeHy7BQw+z9RjNtOgMtOp6aO0yUtm5\nnKrOC3EQOozSMvntP8WNiJmBhM8MIDjSC43G0o9dVbv46Y6f4u7ozivnv8I032kAmLu6aHj1VZpe\nfwMcHQn48Y/xu/UWNDYe9jvVuAgW9qSChTKqpITXL7QsWrs/B5ytq8dkNpvZ9f5z8NJb+DebOJo2\nieTfPUdobKpt+2tHxuZmOnfutEySZ2Ziam0FBwfcUlPxWGZZEOgUHW3zsfjqjmoyqzLJqMxgT/Ue\n9CY9rg6uzJs0jyWhS0gPTWeS+yQADLV11P3pWdq++A8OwcEEPfQQXqsuGbSPJrMkr6K5d+V0PYeq\n2wAI9XblCpMLbpV6YuYGc/6VvrR8+Txl++soMy2kWh+DlAJndwfCEvypDzzGi/VPMCUwhJfPe7mv\nP/31lJdT+9TTdGzZgmNYGMEPP4zH8mV2m8tQwUJRzkb+J/Cv2+DylyDtJqsOKc7ewpHfPUJ4USs1\nk5zx+sWDzLrkZht3dGxJoxHd/v19dx3dhZZFfo5TpvSuJE/Hbd48NC62vRPSG/XsrdnbFzyqOqoA\niPeM5sb9PsR+nofGaMbvth8RcMcdaNzdTztHS1cPO4rq2Xa4jh1F9TR3GdBqBLPCfVkRH0R6pD8l\nn5VScbCJ2ZdEMPeyyO8/0Cu+g3UPoa8qpsLjWkrdr6WouAv0DkjMBEd5EZUURHhiAP5TBk6P7dj5\nDbVPPEHP0aO4L15M8K9/hXOU7UuSq2ChKCNl0MGLsy1DT3duB82Zx7Kb6yv55vf3E7HpEHpnQdNN\nF7H8vj/i6GS/oaLxwlBT0zdJ3rl7N1KnQ7i44D5vXt9dh+PkwcuHjAYpJUdbjpL/+ZsEvfYVvg16\n9sYKPl3pxbSZ6aSHprNo8iJ8nH04XNNumZw+XEdOeTNmCf7uTiyNs9RdWhITiLebI50t3Xz58j4a\nqzpZdkMcCYsHeA9mM+T9A8Pm3/K4m5nPPTz5gecPudBhNRUFLdSXtwPg4etMWKI/EYn+hMb74ej8\n/e+XNBho/uc/qX/xJcx6PX433UTA3Xeh9bRdpWEVLBRlpHY8A9v+ALd8ZZkfGITR0MP2//sNXm/+\nB/cuybHlMSx49C/4h0TasbPjl7m7m67v9vbVrzJUWAoTOsfGfF+/KjV12NlHQ+k+eozaJ/9IZ0Ym\nTlFReP3iAXIjJBmVGWRUZtLc3QQIND3h6FqmYeyIJyEgnhVxlsnp5FCfvvkGgMbjHXz54j66u4xc\ndGci4TMGT0fu6Ongp1vvZ3ftXu5qaeeuLjNi+a9gzu10dpgpO9BI2YFGKg41YdCb0DpomDLNh/CZ\n/oQn+uMdaFmUaGxspO7552n916do/fwI+ulP8b7qSpskFKhgoSgj0VYNL6ZZ1iisfnfQZnlbPqDx\niaeYXKWnIsqTqY/+junzL7ZjRycWKSU9x0q/r5qblQVGIxovLzwWW1Jz3ZcswcFv5HWjTB0dNPz1\nFZreeQeNiwuB992L7w03UN7Ww7bDdWwtrOfbo/UYHSpw8ynC07eEDo4BEOQaxJLQJSwJXcKCkAV9\nGw1VFjbz9f/l4+Co4dJ7kwkMG/wbfm1nLfdsuYcjLUd4dMGjXOWTAOsfhiNbITAeVj5pKdAImIxm\njpe0WIJHfiMttV0A+AS7ET7TctcREuNDz6GD1K5diy4vD5eZM5m05hFck5NH/DMaiAoWijIS/74L\nDnwC93wHfqffIVQfO0D2bx8gek8VzV4ajHf9Pxbf/PBpVWGVMzN1dND5zS46Miypuab6BhACl6SZ\nfXcdLgkJVk3ySrOZ1s8+p+655zA1NuJ51dVUXH0zW2oMbC2s42h9JwBRge59dw9zIvxwctDQoGtg\nZ9VOMioz2H18Nx2GDhw1jswOns38jgvRb/HDJ8idS+9Nwst/8JIfRc1F3L35btp72nl+2fMsnLKw\nt3MSCtfBhl9bkiXiL4WL1oJvxEnHt9Z3UZpvueuoKmrGbJQ4umgJm+5H2Aw//I5n0/7isxjr6/G+\n8koCf/ogjkFBI/3xn0QFC0UZrqoc+PtyWPQAXPC7k17SdbWx/ZmfMemTnWjMUHn5LJY+/AIe3oMP\nSSjWkWYz+oOH6NixnY4dGejz80FKHAIDcV+ajkd6Ou4LF6H1OH1SWrd/PzV/WIt+/346YxL4T/p1\n/KvTi45uI05aDfOi/PrKeof7n358fwazgby6PDIqMqjI0BFbvJAqr2IOzt7Igoi5pIemMyt4Fk7a\nk1Nb91Tv4YFtD+Dm4MbL579MvF/8ACfXw+6XIPNPYDbBop/A4gfA6fQ+GbpNVB5uorT3rqOzxbLx\nVGCoO4FdJbhtex/v7hoC77kLv5tuOusV9ipYKMpwSAlvrISmI3BfDrhY1gyYzWa+/fhFTC+8RkCT\nkWMpQcx47BnCp88d4w6fu4yNjXRkZlomyXd+g7m9HRwdcZs9q++uQ7i5U7j2KRw2rqPNzZu/xV/C\ntqmpBHu7sbw3OCyK8cfNaXjzIWaTmR0fFHEw8zihqZ50LCwmszaDvdV76TH34ObgxvyQ+aSHprMk\ndAl7qvfw6K5HifCK4JXzXxkwNfYkrVWw+THI/xi8psCFv4cZV8Mgd1BSShqrOvqGq2qOtiIlOKHH\nryaPYE0dCXddid8Fy4b1PvtTwUKZ0Jo//IjGN1633wW7O6GjFtwDwdkTg9lAo8aDUv/z6HEMxqQV\nCF9vXNy97denXnqDiRadgY4AR+LPC2XFzEkEef53ZFpJg4Gu3Fw6duygbfsOjEeOAGASGswIPotJ\n5/D5P2BRUjjL44KYHuI54vUJPXojG18roOxAI2krw5l/eRSid6JbZ9TxXfV3lknyqgxqOmv6jps3\naR7PLX8OL6dhLEos2w1fPwQ1+Zb9Qy5+yrIj4RD0HQbKDzZSmt9I+b5auntASBOTNDVc9dcbR/Te\nVbBQJrRj1/4QY13diLeoHBazCWPRehodtNT5TKFO10an8wrMrksR9ODu2UhwXDIajX23rNcZTBRU\ntVLVosPLQcsUHXQJ2OFiQBPpzvLpwayIDyJpivdJ2TvnCiklxf22FM0ua8a/vZGlzUXMdujEc/Vq\nFqSnntWWoid0tnbz1cv7aahoJ/36OBLTB682K6WkuKWYzMpMzNLMLTNuwVE7gqqxZhPkvA1bfg/6\nFph1K6xYY3W1ALNZUlPUSOGHGWA0sPz3q4c+aAAqWCgTllmno3DOXPxvvZWgn52xystZqWivIKMy\ng8z8d9jbWUWP0JDUuJi55Zfh0O1E5Dw/lv1gBm5e9i2/oDeYeDXjKH/dXoKUcNeyaH6cHk1rVQeb\n3jtMW1UnrW6CL7U6jmslAR5OLJ0WxPL4QJbEBuLtOnblrs+W3mBi15EGth2uZ+vhOqpaLFuKTg/x\nYkV8YO+Wor5oRzE4NlV38uWL+9B19HDRHYlEzAwYtXNbRdcM2/4Ie1+zVApYscYSOGxUZfdUKlgo\nE1bnd99R/j83E/rKX/FcvnzoA6xkMBnIrcvtG0o41mpJm4wwmEjvSSOg4Ta6jksmRXmxZPU0gsJt\nW+voVFJKNhTU8IevDlHZrGPVzBB+dUk8ob7fbwgkzZLC72rY/ekRutp6cIvzIt9PsLWskZbeFcez\ne1ccr4gPIibIY8xLYA+lsrnLktp6uI5dRxrpNp7YUjSgtzBfICHettlS9HhxM+teyUfjoOHSe5Ls\n/v/8JLUHYf0v4ViGpZLwxU9B5BKbX1YFC2XCavjbq9Q//zyxu3fh4Ot7dufSNZBZmUlmVSa7ju+i\n09DZlxqZHprO3LzdlO8NoFC3FDdvJxZeHcO0ucF2/4Atqm3nd/8p4JuSRuKCPXns8gQWRg/+DbdH\nbyRrXSn7tlSgddQw6+IITDHubC9uYOvhOg7XWFYLh/q6sjzOEjgWRPvbZZvOoRh6txQ9sedDUW0H\nAOH+bn19nWuHLUWL99ay+e2DeAe4cum9yXgF2CYgDYuUcOgL2LAGWssh4UrLJLjPCPY8sZIKFsqE\nVfG/d9FTVkb01+uGfaxZmiloKCCjKoPMykwKGi0V8U8sukoPTWd+yHyccWHfZ3vI2tqESTiTckEk\nsy4Ox8nFvvMSrV0Gnt9cxLvfluHh7MDPLpzGDXPDcNBat26jpbaLnZ8UU5bfiE+wG4uvjSU80Z/q\nVl3fUM43JQ3oDCZcHDUsjA5geVwgy+ODTrpjsbWGjm52FNaztbCOjKJ62vVGHLWCuZF+LO9d+xAV\nYJ8tRaWU5G4sZ/e/jxAS480ldyXh4j7Ohu4MOvjmL7DzeUDC4gct6baOox/QVLBQJiRpNlO8YCEe\n55/H5LVrrTqmvaedXcd3kVGZwc6qnTTpm9AIDUkBSX0BIs43ru+DqDS/gZ0fF9NapyPCLY9FD/4/\nfKbady8Bk1ny4d4Knt1YSEtXDzfMC+OnF8ThN8LJ2pPe00x/Fl0bi0+QJRjoDSb2HGvqG+opb7Ks\nFp4W7GFJM42z7L9gbYCyhtksKTjeZpmcLqxjf2ULUkKgpzPLe+suLYoJwNPFvh/SZpOZzA+LOZBR\nRczsIM67eToO4+Bua1AtFbDpN1Dwb/AOs9xlJFwxaKrtSKhgoUxI3UePcvSSVYT84ff4XHPNgG2k\nlBxtPWqZnK7KJLc2F6M04uXkxaIpi/oKxfm6nDyE1VLbxc6Piyk70IiPj4nFYi3hP7gZZt9qj7fW\nJ6u0ice+KKDgeBtzI/x47PIEZkw++5Rck9HMvq0VZH1VislkJuW8qcy6OOKkuyUpJUcbOvsCx3fH\nmjCav9/ZbUXvzm7+Hs7Dvn673sDO3mGw7UX11PduKZoc6tM3vDRjsteYZW4Zuk1sfL2A0v0NpF4Y\nxoIro/tSY8e9Y5nw9S+hrgAilljmM4JnjMqpVbBQJqSWTz6hes1viFr31UnlmU+UoD4RIE6UoJ7m\nO82yQGrKEpICk3AYIL311PH9OStDSTr0A7Qu7vDjjCGryo6WmlY9f/z6EJ/nHSfE24VfXzKdS5NC\nRn3opbO1m2//fYTD39YMOQ9z4gN+W2Ed2wq//4BPCvVhxRAf8KcGnr2lJ28puiIuiKW9W4qOta62\nHr56eR/15e0sWT2NmctCx7pLw2cyQvabsPUP0N0Oc26DZb8a9sZcp1LBQpmQjj/yCB2btxC7exc1\nXbV9wWGozW0GcmrmUPzCEOZfEYX7/pdhy+/gf76wbE9qY3qDidd3HuPlbSUYzZIfp0dx17LoYa8u\nHq6ao61kflhEXVm7VRleQw0dLY+zTDwfON428JBW3MBbio615ppOvnxpH12tPVx4+wwik227j7jN\ndTXBtrWQ9Qa4+PSm2t4y4i89KlgoE9KRS1bhNHUqW+9fwLNZzwIQ6hFKeqhlH4LZk2bjrB36m2pt\naRuZHxZRe6yN4EgvlvxwGsGRXtBea6kqG7kUrv+nTd+LlJLNh+r4/ZcHKW/q4qIZwaxZlcBUP/tN\nLEuz5PC31ez+9xF0HQamLwxh/hXRVq0daejoZnthPdv6TUqf4OygYWG0Pyvig1gWF2TX9zQcx0ta\nWPfKfjQawaq7ky2/A+eKmnzL0FTZNzB1Pvxo/YjmMlSwUCYcY3MzxQsWEvjgg9w9eTMAT6Y/SaRX\npNVDNV1tPXz72REO7arG1cuJhVdFEzdv0vdj05/fC/s+gHv2gH+0rd4KJXUdPP7lQTKK6okN8uCx\ny2awONbOi7366dYZ2fvVMfK3VuLgrGXupZEkLpuC1so7gBPprtllzUwP8WRBVACuTuN4Yhgoya5j\n85sH8fR34dJ7k/EOHAepsaNNSsvkt67ZMiw1AtYGC/vmCSrKGejy8gDQJiVQXPwKtybeSpS3ddtK\nmkxm8rdVsvfLYxgNZlIvCGP2JRE4ufb7Fa/eB7nvwYJ7bBYo2vQG/rK5mLd2leLqpOXRSxO4aUH4\nmA/LOLs6sPiaWGYsnszOj4rZ+XExBZlVLPnhNKYmDD3m7ajVMD/Kn/lR47/KrpSSvM0V7PpXCZOi\nvFl1dxIuHuMsNXa0CAGJV9vlUipYKOOGLicXHBwom+KIscjIzIChC6sBlB9sZOdHxTTXdBE2w58l\nP4zFJ/iUYREpYf2vLZOB6Q+Net/NZskn2ZU8veEwjZ09XDdnKj+/MG5EWUW25DvJnUvvS6Y0v5Gd\nHxXxxV/yiEwOYNE1sefEN2+zWbLz42Lyt1USnRbI+bck4DDO74AmCpsGCyHESuAFQAu8JqV88pTX\nnwdO1HNwA4KklD69r4UBrwFTAQlcIqUstWV/lbGly83FZfp09rYXAjAz8MzBorW+i50fl1C6vwHv\nQFdW3ZM0eF2fQ/+Bsp2w6jlw9RnVfueUN/O7LwrYV9nKrHBf3rxlLjND7V+d1lpCCCKTApg63Zd9\nWyrI+rqM93+3h5QLpjJrZcRJe0JPJIYeE5teL+DYvgaSz5vKoh/ETJzU2AnAZsFCCKEFXgYuACqB\nvUKIL6SUB0+0kVI+2K/9fUBqv1O8A6yVUm4SQngAZlv1VRl70mBAl5+Pz+ofkt+QT4h7CAGuA3/w\n9+iN5KwvI3dzOVqthgVXRZO8Yipax0GGeozdsHENBCVA2s2j1ue6Nj1Prj/MpzlVBHs58+fVKVyR\nMnnc12I6wcFRy6yVEcTNC2H3v0vI/rqMw7trWPiDaGJn27/kydnQtffw1V/3U1vaxuJrY0k+b+pY\nd+mcY8s7i7lAiZTyKIAQ4gMTdCWBAAAgAElEQVTgCuDgIO2vBx7rbZsAOEgpNwFIKTts2E9lHNAf\nOoTs7sYtLY38hj8POAQlpaR4by27/lVCZ2sPcfMmseCqaNx9hhjq+fYVaCmDm/49KpU8u40m3vym\nlBe3FGMwSe5eFs09y2Nwd56Yo7oevs5c8KMZJKZPIfOjYja9fpADOyzzGWfac3q8aKnt4j8v7aOz\npZuVdyYSnTo6240qJ7Plb/cUoKLf40pg3kANhRDhQCSwtfepaUCLEOLT3uc3Aw9LKU22664ylnS5\nuQDop0dQtb2K6+OvP+n1+vJ2Mj8sovpIK4Fhnqz88UwmRVkx1NNRBxnPwrSLIXrFWfdz2+E6Hv/y\nIMcaOjl/ehBrViUQEXDm7TonipAYH655eDaHvjnOt58f5aM/7mXG4snMuyIKVw/7lmm3Vs3RVr56\neT8AVz6Yat3vhDIitgwWA93DDpanex3wSb9g4AAswTIsVQ58CNwCnLR1mhDiTuBOgLAw21VlVGyv\nKycXx8mTOaSpBSAxIBGwDC98+8VRDu48jquHI8tvimf6ghDrx6K3/gGMOrjwD2fVv6P1Hfz+y4Ns\nK6wnKtCdt26dw7K4c+8brEYjmLFkCtFpQZZU2+1VlGTXMfeySBLTp6AZR4vtjuTWsemNg7j7OHPZ\nvcmnJzUoo8qWwaISy+T0CaHA8UHaXgfcc8qxuf2GsD4D5nNKsJBSvgq8CpZ1FqPTbcXepJTocnNx\nmzOH/Q370Qot8T7x7Ntawd4vj2HQm0heMZU5qyJwdhtGCmT1fsh5B+bfDQExI+pbR7eRF7cW88bO\nYzg7aHnkkuncvDACJ4fx86FpCy7ujiz54TQSelNtMz8spiDzOIuvicU/1GOsu0fx3lp2flJMcIQX\nq+5OwtVzfN75nEtsGSz2ArFCiEigCktAuOHURkKIOMAX2H3Ksb5CiEApZT2wAlAr7s5RxuPHMdbV\n4ZqaSn79DmJ9Y8n9z3H2ba5g6nRfFv9wGn4hwxzqkRI2/BpcfWHpyFNl//fdbHaWNHDtrFAeWhn3\nX7P39Qn+kz24/CcpHMtrYOcnxXzxl7yx7lKfyOQALrhtBo4qNdYubBYspJRGIcS9wAYsqbNvSCkL\nhBCPA1lSyi96m14PfCD7LSWXUpqEED8HtghLSkY28Hdb9VUZW105lvkKl9QUDux/mZURKyld10B4\noj+r7kkaWVbO4a+gNBMuedYSMEbSrx4ju4828uP0KH51yfQRneNcIIQgKjWQsBl+HMmtx9Cv7MdY\ncXZ3JDot6Jzce3y8smn6hpRyHbDulOcePeXxbwc5dhOQZLPOKeOGLjcX4eZGzSRn2rPaSXBJorZe\nR+LSKSMLFCdSZQPjLXsZj9C+ilZMZsm8qLOr6nmucHDSEjdv8MKNyrnt3B54VSaErtxcXJOTONBi\nyaoObo0EYHLsCBfPffcqNB+Di544q1TZnPJmANLCzm5rV0U5F6hgoYwpU0cn3YWFuKWmsr9+P+6O\n7hirnHB00RIwkonUzgbY8TTEXgQx551V37JKm4gJ8sDHTU2eKooKFsqY0ufvB7MZ19RUDjQcINE/\nkeriVkKifUaWprltLRi6zjpV1myW5JS3MDtc3VUoCqhgoYyxrpwcEAJNYjyFzYUkuifTXNPF5NgR\nLK6qLYDst2DOHRA47az6dbShg1adgTQVLBQFUMFCGWO63DycY2IoMlRhNBsJ70oAYMq0YX5ISwnr\nfwUu3rD0F2fdr6xSy3zFLBUsFAVQwUIZQ9JkQpeXh2taGvkN+QC41wfg4KgZfk2iovVwbAcs+/VZ\n70kMkF3WjK+bI1HnSCkPRTlbKlgoY6a75Ajmjg5cU1PIb8hnkvskmo51MynaG+1wVkgbe2DDIxAQ\nB7NHnirbX3ZZM7PCfSdU5VVFsSUVLJQxc6J4oFtqKvn1+SR7pdFY1TH8lNm9f4emI3DRWtCe/Y5o\nTZ09HG3oZFa4Wl+hKCeoYKGMGV1uDlp/f9qDPKjsqCSuOxXkMNdXdDbC9qcg5nyIvWBU+pVTpuYr\nFOVUKlgoY6YrNw/X1BQKGgsA8GsKReMgCI70sv4k25+Ang64cO2o9SurrBlHrSBpHO92pyj2poKF\nMiaMDQ0YystxS7VMbmuEBmOVI8ERXjg4WlkYrvYgZL0Bc26DoPhR61tOWTMzJnvjYm0/FOW/gAoW\nypjo6p2vcO2dr4jzmE5jZZf1KbMnqso6e8KyX41av3qMZvZVtqghKEU5hQoWypjQ5eYhHB1xTphO\nfkM+yaYFSLNkcoyV8xXFG+HoNkugGIVU2RMKjrfSbTSrlduKcgoVLJQxocvJwSUxkYruGtp62pjS\nHotGI5gUbcU8gclguavwj4U5t49qv7J7J7fVym1FOZkKFordmbu70RcUWIagehfjOdZ6ERjuiaOz\nFfMEe1+DxpJRS5XtL7usmVBfV4K9/rs2OVKUoahgodidvqAAaTDglmYJFp7Cm7ZKg3Ups11NsP2P\nEL0CYi8c1X5JKckqa1ZDUIoyABUsFLs7sRjPNSWF/Pp85miWYDZJ64LF9j9Cd7tlr4pRXl1d2ayj\nvr1bTW4rygBUsFDsris3F8ewMEy+nhxuPkx010yEgJChJrfrDsPe12H2jyBo9Lc5ze5bjKdWbivK\nqVSwUOxKSokuJxe31FQKmwoxmo14NAQTMNUTZ9chdrXb+Ag4eViKBdpAdlkz7k5a4iYNs4ihovwX\nUMFCsStDeTmmpqa+yW2NWUv3cc3QKbPFm6BkMyz7Jbj726RvWWXNpIb5otWo4oGKcioVLBS76srp\nna9Is2yjGmdMxmyUTJ52hmBxIlXWL9qysZENtOsNFNa0qfkKRRmETYOFEGKlEKJQCFEihHh4gNef\nF0Lk9f4pEkK0nPK6lxCiSgjxki37qdiPLjcXjYcHzjEx5Dfkk9gzF4CQmDOsr8h6AxqKLKmyDrbZ\nDzuvogWzVMUDFWUwQwwSj5wQQgu8DFwAVAJ7hRBfSCkPnmgjpXywX/v7gNRTTvN7YIet+qjYny43\nF9eUFFp72qhor+CylnB8Jrvj6jFIEOhqgm1PQNQymLbSZv3KLmtGCEgNG2Z5dEX5L2HLO4u5QImU\n8qiUsgf4ALjiDO2vB94/8UAIMQsIBjbasI+KHZna2uguKcG1d32FkBqocWPKmVJmdzwF3W02SZXt\nL7usmbhgTzxdRneRn6KcK2wZLKYAFf0eV/Y+dxohRDgQCWztfawB/gQ8ZMP+KXam27cPpLRsdtSQ\nT1BnGGYDhAwWLOoL4bu/w6xbIHiGzfplMktyy1XxQEU5E1sGi4G+BspB2l4HfCKlNPU+vhtYJ6Ws\nGKS95QJC3CmEyBJCZNXX159FVxV70OXmgkaDa1IS+xv2k2iwzFcMuhhv4xpwcoflj9i0X0W17XR0\nG5kdoYKFogzGlsGiEpja73EocHyQttfRbwgKWADcK4QoBZ4F/kcI8eSpB0kpX5VSzpZSzg4MDByd\nXis205WTi3N8HMLNjQMNBwjriMcn2A13b+fTG5dstlSWXfoLcA+wab+yTizGC1OL8RRlMFYFCyHE\nv4QQq3qHh6y1F4gVQkQKIZywBIQvBjh3HOAL7D7xnJTy/0kpw6SUEcDPgXeklKdlUykThzQa0e3f\nj1tKKhXtFbTp23Cp9x38ruLb/wPvqTD3xzbvW05ZM4Gezkz1c7X5tRRlorL2w/8V4AagWAjxpBBi\nyG3JpJRG4F5gA3AI+EhKWSCEeFwIcXm/ptcDH0gpBxuiUs4B+sJCZFcXrmlp7G/Yj1/XZGS3ZuBg\nYdBBaSbEr7JZqmx/2WXNzArzRdhwAl1RJjqrUmellJuBzUIIbywf7puEEBXA34H3pJSGQY5bB6w7\n5blHT3n82yGu/RbwljX9VMYvXW4eAG6pKeRXvkN4h6W204DBonQnGPUQc4HN+1XXrqe8qYub5ofb\n/FqKMpFZPawkhPAHbgFuB3KBF4A0YJNNeqacU3S5uTgEB+MweTIHGg4Qo0vCK8AFT78B9o0o3gQO\nLhCxyOb9yjkxX6EmtxXljKyds/gUyATcgMuklJdLKT+UUt4HeNiyg8q5oSs3B9fUVAxmA4caD+Hb\nPGXwelAlmyBiCTjafg4hq7QZJwcNMyZ72fxaijKRWXtn8ZKUMkFK+UcpZXX/F6SUs23QL+UcYqip\nwXi8GrfUFAqbCvHo9EfoHQauB9V4BJqOQqzth6AAssubSQ71xtnBih36FOW/mLXBYroQou9fthDC\nVwhxt436pJxj+jY76p3cDmmLBgaZryjZbPlvzPk275feYOJAVavab1tRrGBtsLhDStlX5E9K2QzY\npvyncs7pys1FuLjgEh/PgYYDRHbOwN3HGa+AAYaZijeBXxT4R9u8X/lVrRhMkllhKlgoylCsDRYa\n0S+vsLdIoO1zGpVzgi43D9eZMxGOjuTX5xPSFs3kWJ/TU1VPpMzaIQsK+u+Mp4KFogzF2mCxAfhI\nCHGeEGIFltXW623XLeVcYdbp0B86hGtqKq3drbTUdeGgdxkkZfYbS8qsneYrskqbiQxwx99jgBXk\niqKcxNoS5b8EfgzchaXm00bgNVt1Sjl36PLzwWjENTWFfQ35ffMVUwaa3C45kTK72Ob9klKSU97M\nivggm19LUc4F1i7KM2NZxf2KbbujnGt0J3bGS0khv/wDJrfF4OLpgE+w2+mNizdZAoUdUmZLG7to\n6uxRQ1CKYiVr11nECiE+EUIcFEIcPfHH1p1TJj5dbi5OUVE4+PqSX5/P1I44psQOUFqj6Sg0HbHb\nfEVWaROg5isUxVrWzlm8ieWuwggsB94B3rVVp5RzgzSb0eXl4ZqWipSSI5XluOq9Bp6vKO5NmbXT\nfEVOeTNeLg7EBKo1pYpiDWuDhauUcgsgpJRlvfWcVtiuW8q5oOfYMUytrbilplLZXolHg2V+YHLs\nAN/mSzaBb6RdUmbBMrmdFu6LRqOKByqKNawNFvre8uTFQoh7hRBXAWpmUDmjvsV4vTvjhbRF4+Aq\n8J/sfnJDgx6OZdrtrqK1y0BxXYdaX6Eow2BtsHgAS12o+4FZwI3AzbbqlHJu6MrJRevtjVNkJPkN\n+Uxpi2VKrB/i1G/zZTvBqLPbfEVOhSoeqCjDNWSw6F2A90MpZYeUslJKeauU8gdSym/t0D9lAtPl\n5uKamooQgkOVJXjpAwidNsAHdPFm0DrbJWUWILu0Ga1GkDJ1kEKGiqKcZshg0bsv9iyhdoZRhsHY\n3EzPsWO4pqVhMBnoKLVsrz5wPajelFmnAdJpbSC7rJmEEC/cnKxdZqQoirX/WnKBz4UQHwOdJ56U\nUn5qk14pE17/zY6KmosIao1AOEkCQk/JPmo6Bo0lMOd2u/TLYDKTV9HC6jlTh26sKEofa4OFH9DI\nyRlQElDBQhmQLjcXHBxwSUxkf9lnhLRFExTpgUZ7ys1sX5VZ+8xXHK5uR2cwqUqzijJM1q7gvtXW\nHVHOLV25ObgkJKBxdaWg4jAhuqVETg8+vWHxJvCNsFvKbHaZZTHebBUsFGVYrAoWQog3sdxJnERK\n+aNR75Ey4cmeHvT5B/C9bjUANUfaCGGA9RUGPRzLgNQbwU5TYlllzYR4uzDZx/YlRRTlXGLtMNSX\n/f7uAlwFHB/97ijnAv3hw8jublxT02jtbsWhxgsczASFe57csOwbS8qsndZXgGXPbVXiQ1GGz9ph\nqH/1fyyEeB/YbJMeKRNeV04OYFmMt7ehgJC2aDynOqB1GGC+Quts2W/bDo636DjequcOFSwUZdis\nXZR3qlggbKhGQoiVQohCIUSJEOLhAV5/XgiR1/unSAjR0vt8ihBitxCiQAixXwixeoT9VMaALjcP\nx8mTcQwOYl/VAfy7JhMVP+n0hsWbIGKRXVNmQRUPVJSRsHbOop2T5yxqsOxxcaZjtMDLwAVAJbBX\nCPGFlPLgiTZSygf7tb8PSO192AX8j5SyWAgxGcgWQmzov7WrMj5JKdHl5OA2bx4AZYV1TCXm9Mnt\n5lJoLIbZ9pv2yi5rxtVRy/QQL7tdU1HOFdYOQ3kO3eo0c4ESKeVRACHEB8AVwMFB2l8PPNZ7vaJ+\n1z4uhKgDAgEVLMY5Q9VxjPX1uKamIKWksxykxkxwxCkf0MWbLP+143xFdlkzyVO9cTw1fVdRlCFZ\nu5/FVUII736PfYQQVw5x2BSgot/jyt7nBjp/OBAJbB3gtblY9vs+MsBrdwohsoQQWfX19UO/EcXm\nThQPdEtLo6qjCr+mKTiFmHBw0p7csGQz+ISDf4xd+tXVY+RgdRuzw/3scj1FOddY+xXrMSll64kH\nvcNBjw1xzEC5kKel3/a6Dvikt7TI9ycQIgTLvhm39u7Wd/LJpHxVSjlbSjk7MDBwiO4o9qDLzUHj\n5oZzbCz7qvIJ6AwdPGU29gK7pczmVbRgMks1X6EoI2RtsBio3VBDWJVA/5oKoQyebnsd8H7/J4QQ\nXsBXwBpVtHDi6MrNwyU5CeHgwOFDZWjQMmNmxMmNyneBoctuq7bBkjILkBqmigcqykhYGyyyhBDP\nCSGihRBRQojngewhjtkLxAohIoUQTlgCwhenNhJCxAG+wO5+zzkB/wbekVJ+bGUflTFm6uiku7AQ\nt9Q0ABqP6jALM6Ex/ic3LN4MWieItE/KLFjmK2KDPPBxc7LbNRXlXGJtsLgP6AE+BD4CdMA9ZzpA\nSmkE7gU2AIeAj6SUBUKIx4UQl/drej3wgZSy/xDVD4F04JZ+qbUpVvZVGSP6/fvAbMY1NRWD2YBD\nrRcyoAtH51PnKzZB+CJwch/4RKPMbJZkq8V4inJWrM2G6gROWydhxXHrgHWnPPfoKY9/O8Bx7wHv\nDfd6ytjqys0FIXBNSaagrpCA9qn4Tz9lTqK5DBqKYNYtduvXkfoO2vRGFSwU5SxYmw21SQjh0++x\nrxBig+26pUxEupxcnGNj0Xp6si+/CK10IH7GKWs3S3pTZu04X5GlFuMpylmzdhgqoP+COCllM2oP\nbqUfaTKh27cP11TLusqqoiYkkhmJUSc3LN4MPmEQEGu3vmWXNePn7kRkgH2GvRTlXGRtsDALIfq+\nIgohIhg8DVb5L9RdUoK5owO3NEuw6KlypNunFRc3x+8bGbstKbMx9kuZBUsmVFqYL2qzR0UZOWur\nzj4C7BRC7Oh9nA7caZsuKRPRicV4rqmpNHe24N0cjNPMrpMble0CQ6ddV203dnRztKGTa2ernfEU\n5WxYdWchpVwPzAYKsWRE/QxLRpSiAJZgoQ0IwHHqVLLyD+AgnQiPP2WhZMmJlNl0u/Urp9wyejo7\nQs1XKMrZsLaQ4O3AT7AsrMsD5mNZF7HiTMcp/z26cnJxS01BCEHJoWrAn9kpCSc3Kt4E4QvtljIL\nkFXWhKNWMHOK99CNFUUZlLVzFj8B5gBlUsrlWKrDqmJMCgDG+noMFRW4pljmK9pKjbR7NBDk128x\nXks5NBTaNQsKLPMVMyZ74+KoHbqxoiiDsjZY6KWUegAhhLOU8jAQZ7tuKRNJ14n5irRUTEYTzg0+\naCbrT240BlVme4xm9lW2qv22FWUUWDvBXdm7zuIzYJMQohm1rarSS5ebh3BywmXGDA4WleJocsY3\n5pSq9iWbwTsMAqbZrV8HjrfSYzSr9RWKMgqsXcF9Ve9ffyuE2AZ4A+tt1itlQtHl5uKSmIjGyYkD\n+UcAB2YmRX/fwNgNR3dA8mq7p8yCWoynKKNh2LvASCl3SCm/kFL22KJDysRi7u5GX1CAa6qldFdd\nSQetrvXMDJv+faPy3ZaUWTvPV2SVNjPVz5UgLxe7XldRzkVqyzDlrOgLCpAGA26pqZjNElnthj6o\nCUdNv8V4xZvsnjIrpSS7vJlZYequQlFGg7VzFooyIF1ODmBZjFdb0YKD0QnvcMeTG5VshrAF4Oxh\nt35VNuuob+9mVsT43RnPYDBQWVmJXq8furGinCUXFxdCQ0NxdHQcuvEAVLBQzkpXbh6O4WE4+PuT\nvysLgGkz+u2e21IB9Ych9Ua79iurrAlgXN9ZVFZW4unpSUREhCpFotiUlJLGxkYqKyuJjIwc0TnU\nMJQyYlJKdLm5fZsdlRXW0+bcSGrUzO8bjUGVWbAUD/RwdiBukufQjceIXq/H399fBQrF5oQQ+Pv7\nn9VdrAoWyogZysowNTXhmppqCRwVgkafCqZ49LuzKN4M3lMh0L7LcrJKm0kN80GrGd8fxCpQKPZy\ntr9rKlgoI9aVmweAa2oKzdVdaLudcJ5q/P6X0tgDx3ZAzPl2TZlt1xsorG0nbRwPQY038+bNIyUl\nhbCwMAIDA0lJSSElJYXS0tJhnefTTz/l8OHDw77+4sWLycvLG/ZxJzz77LP885//HPHx9nDttddy\n9OjRAV9bv349aWlpzJw5k1mzZrF9+/YB2zU2NnLeeecRGxvLRRddRGtrqw17fDIVLJQR0+XmovH0\nxDkmhqOHagAIndavxEf5bujpsOuqbYC8ihakVMUDh2PPnj3k5eXx+OOPs3r1avLy8sjLyyMiImJY\n5xlpsDgbBoOBd999l9WrV9v1usP1v//7vzzzzDMDvhYUFMRXX31Ffn4+b7zxBjfddNOA7dauXcvF\nF19McXExS5Ys4emnn7Zll0+igoUyYrrcHFxTUhAaDUWHKulwaiEpKv77BiWbQONo15RZsAxBaQSk\nTPUZurEypK+//poFCxaQlpbG6tWr6ezsBOChhx4iISGBpKQkfvnLX5KZmcm6det48MEHR3RXcsJ7\n773HzJkzSUxM5Ne//nXf83/729+YNm0ay5Yt4/bbb+eBBx4AYNOmTcyZMwet1lL/69tvvyUpKYmF\nCxfy0EMPkZJiWQN05MgRlixZQmpqKrNmzWLPnj0AbN68meXLl3PNNdcQGxvLmjVreOedd5gzZw5J\nSUl97+PGG2/knnvuYfny5URHR5ORkcHNN99MfHw8t912W18/77zzTmbPns2MGTN4/PHH+55ftmwZ\n69evx2Qynfae09LSCAkJAWDmzJl0dHRgMBhOa/f5559z8803A3DzzTfz2WefjehnPBIqG0oZEVNb\nG93FJXhdcglSSpqPdlPteYTEwPO+b1S8GcIXgLN9J5lzypuJm+SFp8vIUgTHwu/+U8DB422jes6E\nyV48dtmMszpHXV0dTz75JFu2bMHNzY21a9fywgsvcNttt7Fu3ToKCgoQQtDS0oKPjw+XXHIJ11xz\nDVdeeeWIrldZWcmaNWvIysrC29ub888/ny+//JLk5GSefPJJcnJycHd3Z9myZcydOxeAb775hlmz\nZvWd49Zbb+Xtt99m7ty5/PznP+97PiQkhE2bNuHi4sLhw4e5+eab+wLGvn37OHToEN7e3kRERHD3\n3Xezd+9e/vSnP/HSSy/x7LPPAtDa2sq2bdv417/+xWWXXcbu3buJj48nLS2NAwcOkJiYyJNPPomf\nnx9Go7EvCCUkJKDVaomIiODAgQMkJycP+jP46KOPmDdv3oApro2NjQQGWkr/T5kyherq6hH9nEdC\n3VkoI6LLOzFfkUprvQ66HOgJbsbbubcUeGsl1B+yexaUySzJLW9hVri6qxgNu3bt4uDBgyxcuJCU\nlBT+8Y9/UFpaip+fHxqNhjvuuIN///vfuLuPTtn5PXv2sGLFCgICAnB0dOSGG24gIyOj73lfX1+c\nnJy45ppr+o6prq7u+wBtaGigp6enL5DccMMNfe26u7u57bbbSExM5LrrruPgwYN9r82bN4/g4GBc\nXFyIiorioosuAizf8vvfIV122WV9z0+ePJmEhAQ0Gg0JCQl97d5//33S0tJIS0vj0KFDJ10nKCiI\n48cHL6uXn5/PmjVreOWVV6z6edkzQcKmdxZCiJXAC4AWeE1K+eQprz8PLO996AYESSl9el+7GVjT\n+9ofpJRv27KvyvB05eaCVovrzJmU5lpqMAVEuX3fYAyqzAIU1rTT0W2ccPWgzvYOwFaklKxcuZJ3\n3333tNeysrLYtGkTH3zwAa+88gobN24c9Dz9P8CvvvpqHn300UGvN5znAVxdXftSQs/U7k9/+hNT\np07lvffew2Aw4OHx/SJRZ2fnvr9rNJq+xxqNBqPReFq7/m36tysuLuaFF17gu+++w8fHhxtvvPGk\ndFW9Xo+rqyuffPIJf/jDHwB46623SElJoby8nKuvvpr33ntv0LUQ/v7+1NfXExgYSFVVFZMmTRr0\n/Y42m91ZCCG0wMvAxUACcL0Q4qTdcKSUD0opU6SUKcCLwKe9x/oBjwHzgLnAY0KIifWv/xyny83D\nJS4Ojbs7Rw/VoHNoJz66X/HAks3gFQqB8YOfxAayyy2Ba3b4+F25PZEsXLiQHTt29GXxdHZ2Ulxc\nTHt7O21tbVx66aU8//zz5PaWqff09KS9vf208zg5OfVNmg8WKADmz5/Ptm3baGxsxGg08sEHH7B0\n6VLmzZvHtm3baGlpwWAw8Omnn/YdM336dEpKSgAIDAzE0dGRrCzLAtEPPvigr11rayshISEIIXj7\n7bfPGFhGqq2tDU9PT7y8vKiurmbDhg0nvV5cXMyMGTO45ppr+n4eKSkpNDc3s2rVKp599lnmz58/\n6Pkvv/xy3n7b8r357bff5oorrhj19zAYWw5DzQVKpJRHe4sOfgCc6Z1dD7zf+/eLgE1SyiYpZTOw\nCVhpw74qwyCNRnT79+Oaatns6HhxC8e9jpAU2LsYz9gDR7dDrH1TZgGyS5sI9HQm1NfVrtc9VwUH\nB/P666+zevVqkpOTWbhwIUVFRbS2trJq1SqSk5NZsWIFzz33/9s79/iqqmvff0deJJAQIOFlAoFI\neCaUYAoUrLXVCiJCr+UKtvb41vai5X4sHOvrKBxBiaXVU22tAsXezz2otahU8SBQ0R4voiEBA0Fh\n80gMSUgISSDknT3vH2tlG0Le2XuvkIzv57M/2WutudYcc6+9M9acY8zf/C0At9xyC6tXr+50gDs2\nNpaVK1dy9dVXM2XKFIpUyN4AACAASURBVGbMmMENN9zAyJEjWb58OdOmTeO6665j0qRJREZaQ55z\n587lo48+8lxjw4YN3HHHHcycOZOAgABPufvvv59169YxY8YMsrOzL+gZeIupU6cyceJEEhMTueee\ne5g1a5bnWF5eHpGRkZ4hs8Y8//zzHD9+nCeeeMKTtlxcXAxYMZiGtOJHHnmE9957j4SEBD7++GOW\nL1/u9Ta0iDHGJy9gIdbQU8P2z4AXWigbB+QDgfb2MuCxRscfB5a1Vt8VV1xhFP9QceCAyRo33pS+\n+64pO11hXrhvp/npMw+Ymroaq8Cxj4x5or8xWX/3u21Xrtlpfv5/0vxeb2fIyspy2oRLinPnzhlj\njKmpqTHXX3+92bJli+fYjTfeaI4ePXpBOWOMeeqpp8yDDz7oX0NbIDU11WzcuNFRG5r7zgFpph3/\n033Zs2jukbKlft9i4E1jTENOWbvOFZF7RSRNRNKKinSVV39RmW4NOfRNTib/SCkAYSPdBAfa2RtH\n7JTZ+O/51a7Cs1V8fabykotXKO3j8ccfJzk5mcmTJzNu3DjmzZvnObZmzRpP4HjLli1MmTKFxMRE\ndu/ezcMPP+yUyRcQFRXFrbf6VyPNm/gywJ0LjGi0HUvLq+stBpY0OffqJufuanqSMeZl4GWAlJQU\n7w9AKs1SmZFB0NChBA0fTu6OQ1QHVTAmvtGtdu2AkTP8njK7117saKo6ix7J7373uxaPTZjwzfop\nP/nJTy7Iguou3HnnnU6b0CV82bP4HEgQkdEiEoLlELY0LSQi44CBwO5Gu7cB14nIQDuwfZ29T+kG\nVOzLIGxqMiJC9ldF5EUcJXFwonWwLBcKs/yeBQWWswgJCiDxski/160oPR2fOQtjTB1wP9Y/+UPA\nG8aYgyKyUkTmNyp6C/CaPXbWcO4Z4N+xHM7nwEp7n+IwtQUF1OXl0zc5mfNl1VQW15Pf/yiToydb\nBVw7rL9+nl8BkJZdwrdiIwkJ0ulDiuJtfDrPwhizFdjaZN+/Ndl+soVzNwAbfGac0ikq7RTJsOSp\n5NrxivKoQmIjYq0Crh3QPwaGTGjpEj6hqraeg3ll3HVlvF/rVZTegj6CKR2iIj0DCQ0ldPw48g6X\nUhdYQ8woe02G+lo45n+VWYAvcsuorTca3FYUH6HOQukQlRkZhCUlIcHBfH34DHkRR0kaYscrvt4D\n1Wcdi1cATB2pMh+dQSXKfU9rEuWFhYVcffXV9OvXzyOQ2BwLFy703Ju4uDhSUlIAcLlchIWFeY4t\nWbKkxWt0FhUSVNqNu6KCqkOHiLr7birP1VBWUEneSBc3Dv6xVeDIdggIgtH+TZkFy1nER/cjKtz7\nE616Aw2Cehs3biQtLY0XXnihU9fZvHkzAQEBjB/vv5n7DRLl6fZ68N2VBony5nSfGkQaMzIyPLPR\nm+PNN9/0vF+6dClDhw71bI8bN65LDrcttGehtJvKzANQX09Y8hTyXFa8Ir+/i6Roe+a2aweM/A6E\n9verXcYY0nNKNGXWR6hEudUOX0qUh4eHM2vWLEJDQ9v1Gbndbv7617+yePHiDnyyXUN7Fkq7aQhu\n950yhbwPCnEH1hE2XCyl2bN5cOoAXLvC73YdP32eM+drLu14xfu/hoJM715zWBJc/0zb5VpBJcr9\nL1HeHnbt2sXIkSOJj/8mocPlcpGcnExkZCSrV69m5syZXaqjKdqzUNpNRUY6IZdfTuCAAeQdLuV0\n/6+ZNMTWhmxImXUgXpGW3SAeeAk7i26KSpT7V6K8vWzatIlbbrnFsx0bG0tOTg4ZGRmkpqayaNEi\nysvLu1xPY7RnobQL43ZTuW8/ET+8luqKWk6fLCc75hA/jLbEBDmyHSIugyETW7+QD0jPLqF/aBCX\nDw5vu3B3pYs9AF9hVKLcbxLl7aW2tpZ33nnngiGu0NBQzxDWtGnTiIuLw+Vydei6baE9C6Vd1Bw/\njrusjL7JU8l3lYGBvP4uazJefa1jKrNgBbeviBtIQID/6+7pqER5x+isRHlH2LZtG0lJSZ5lWAGK\nioo8sRCXy8WxY8daXBOjs2jPQmkXFXamSVhyMgczSjEBbkr65zFu0Dj4+jMrZdaBWdulFTUcKSxn\nwZTL/F53b6CxRHlNTQ0Aq1evJiwsjJtuuonq6mrcbvcFEuX33Xcfa9eu5e2332bUqFEdqq+xRLkx\nhhtvvJEbbrgBwCNRHhMTc5FEeeMAc4NEeUREBFddddUFEuULFy5k06ZNXHvttT6XKI+Pj2+3RHlD\n2ysqKqitreXNN99k586djBs3jjvuuIOlS5d6nMprr712wRAUwIcffsiKFSsIDg4mMDCQV155xdNu\nr9EeadpL4aUS5b7l5MOPmK+mzzBut9u88fTn5ulH/tPc8u4t1sHtTxizYpAxlWV+t+sfh06ZuIfe\nNZ+4ivxed1dRifKOoRLlXae7SpQrPYjK9HTCkpOpra6nKOcsx/se/CZl9sgOGDHD7ymzAGnZZwgM\nEKaM0Ml4PR2VKHcWHYZS2qSupISaEyeIvOkmCo6VYdyQE/4lNw6+Hc7mw6lMuPZJR2zbm13CxOH9\n6RuiX+WejkqUO4v2LJQ2qcywZoX2nZpM3uFSEENBxHEruO2gymxtvZv9X5dd2vMrFOUSQZ2F0iaV\nGekQHExoYiJ5rlJqo87Rr28YIyJGgMtOmR06ye92Hco/S2VtvToLRfED6iyUNqnIyCB04gTcAcGc\nOnGWk/2PkBidiLjr4eguGHONYymzgDoLRfED6iyUVjE1NVRlHqDvlGQKjp/FXWc4FJJuDUHlfgbV\nZY7M2gZr5vZlkaFcNiDMkfoVpTehzkJplapDhzDV1YQlJ5NnL3aUH+EiMTrxG5XZ+KsdsS09W8UD\nvYVKlPseb0iUP/bYY8TExHjuT9NJf75EU0iUVqnwrIyXTN5/5iKDq6kJqrLSZl2PwIjpEOr/Na/z\nSivJL6tSPSgvoRLlvscbEuVgTU5szaH4Cu1ZKK1SmZ5BcEwMAVHRFBw7S8nAk4yIGMHA2mpLJXXM\ntY7YleaJVwxypP7ehEqUW+3oThLlTqA9C6VFjDFUZKTTb8Z3KDxxlvpaN1/2Sbd7Fc6pzII1BBUW\nHMj44RGO1O9t1ny2hi/PdHz4pjXGDxrPQ9Me6tI1VKK8+0mUP//882zYsIFp06axdu1a78t6tID2\nLJQWqT15kvqi0xcsduRxFke2Q8RwGJroiG17s0uYMmIAwYH6FfYlKlHevSTKH3jgAVwuF/v27SMq\nKorly5d3+lodxac9CxGZAzwPBALrjDEX6TCLyM3Ak4AB9htjfmLvTwVuwHJo24Glto6J4ic8ix0l\nJ5O3vZQ+gw1VwedJGjQRjj0CE250JGX2fHUdWfln+cX3Lvd73b6iqz0AX2FUorxbSZQ3Xkb1nnvu\nucBp+hqfPZaJSCDwInA9MBG4RUQmNimTADwMzDLGTAL+t71/JjALmAwkAt8G/L+wcy+nMiODgL59\nCb48gfyjZVQMLiYoIIjxFeVQVebIrG2A/bml1LsNV4zS4LavUYnyjuFrifL8/HzP+7feeovERP/1\n7H3Zs5gGuIwxxwBE5DVgAZDVqMw9wIvGmBIAY0yhvd8AoUAIIEAwcMqHtirNUJGeQdiUb3E6r4La\n6nqy+2UxfuB4+hzfBRLoWMrs3hNWcHvqCHUWvkYlyjuGryXKf/WrX5GZmYmIEB8fz0svveT1NrRI\ne6RpO/MCFmINPTVs/wx4oUmZt4FU4BPgU2BOo2O/AUqBMmBVW/WpRLl3qTt3zmRNmGgK/+P3Jn1b\ntnnhvp3mqg0/ME/tfsqYP15pzPo5jtl224Y95tq1uxyr31uoRHnHUInyrtNdJcqbG8xu2u8LAhKA\nq4FbgHUiMkBExgATgFggBviBiFx1UQUi94pImoikFRUVedX43k7l/v3gdlvzK1yl9I0O4kxAIZMj\n4qDgC2tVPAdwuw3p2SWk6BBUr0Mlyp3Fl8NQucCIRtuxQNM0gFzgU2NMLXBcRL7iG+fxqTGmHEBE\n3gdmAB83PtkY8zLwMkBKSooGv71IZcY+EKFPUhL5m/cRcLmVW590zhoCcipe4Soq52xVHVNHqrPo\nbahEubP4smfxOZAgIqNFJARYDGxpUuZt4PsAIhINjAWOATnA90QkSESCsYLbh3xoq9KEyowM+owd\nS9k5obqijlORx4kIiSAuJw3Ch8GwJEfsahAPTBmlk/EUxZ/4zFkYY+qA+4FtWP/o3zDGHBSRlSIy\n3y62DSgWkSzgQ2C5MaYYeBM4CmQC+7FSav/uK1uVCzH19VTu20dY8hROHrbmV2QG7yEpKhE59qE1\na9uBlFmAtBMlDOoXwqiovo7Uryi9FZ/OszDGbAW2Ntn3b43eG+BB+9W4TD1wny9tU1qm2uXCff48\nfadOJf9IKeGD+nCweh/3hMy1UmYdilcApOeUMHXkQMQhZ6UovRWd/qpcRKUtyBY6xZq5HTrCjdu4\nmVxeaqfMft8Ru4rLqzl++rwGtxXFAdRZKBdRkZFBYHQ05UGDqDxXy9koayJQ4slMGDENwgY4Ypcu\nduQ7VKLc93hDovzBBx9k3LhxTJ48mR//+MeUlZUB4HK5CAsL89y3JUuWeN1+dRbKRVRm7KNvcjL5\nLuuLeCTsC2L6DmNQvnMqswB7c0oIDhSSYvwvid7T2bNnD/v27WPlypUsWrTIM7u4o5PqOussukKD\nRPmiRYv8Wm9HaZAob44GkcY1a9a0eo3Zs2dz8OBBvvjiC0aNGkVqaqrn2Lhx4zz37cUXX/Sq7aDO\nQmlCXVERtV9/bc2vOFxCv8gQMqr2MDnEfpp3SGUWrJnbiTGRhAYHOmZDb0Qlyq12dAeJ8tmzZxMU\nZIWaZ8yYQW5ubkc/3k6jEuXKBZzbuRPAUpp9rZSo+DAKKgpIqg2G8KEwbLIjdlXX1fPFyTL+ZUac\nI/X7moLVq6k+5N0n8j4TxjOs0T/bzqAS5d1PoryBDRs2cNttt3m2XS4XycnJREZGsnr1ambOnNnl\nOhqjPQsFgJrck+T+cikFT64gZMzl1AyJ53xZDdVDrdTZpPwvHU2ZPZh3lpo6twa3/YxKlHcvifIG\nVqxYQXh4OIsXLwYsXamcnBwyMjJITU1l0aJFlJeXd7mexmjPopfjrqykeN16itetAxGif/kAUXfe\nyZd7zwCQG36YoLOBjC8/42y8okE8sIcGt7vaA/AVRiXKu5VEOcD69ev54IMP2GmPAgCEhoZ6hrCm\nTZtGXFwcLperQ9dtC3UWvRRjDOe2beNUaip1efn0nzuXIcuXETx8OAB5R0oJiwgmo24vY4P6E4rA\n5c6kzIKVCTVyUF+GRHTfZSd7IjNnzmTp0qUcO3aM+Ph4zp8/T15eHsOGDaOqqop58+Yxffp0Jk60\nVh9oS6K8LWbMmMHy5cspLi4mMjKS1157jWXLlpGUlMRDDz1EaWkp/fr1Y/PmzaSkpAAtS5SnpKRc\nJFE+ZswYv0uUz5kzx3O8QaJ88ODBnVqL4r333uO3v/0tH3/88QXxjaKiIgYNGkRgYCAul4tjx44x\nevRor7SpAXUWvZCqrw5zatUqKj77jD7jxxOzZg19v/3tC8rkHS5l+JhIDpw5wLzKOoidBmHOPNUb\nY0jLLuG7CdGO1N+bUYnyjuFrifIlS5bgdru55pprAJg1axYvvvgiH374IStWrCA4OJjAwEBeeeUV\n7y+32h5p2kvhpRLlbVNXUmLyV6w0WRMmmq+mTTdnNm0y7rq6i8qVna4wL9y30+x4N90kbkw07zw7\n3JiPUh2w2CL79HkT99C75i+7Tzhmgy9QifKOoRLlXacrEuXas+gFmPp6St94g6Lnnqf+3DkGLl7M\n4F8+QOCA5ifX5R+xgtrFA3LgNCRWVzumMguwN8eKn6T00HiF0j4ef/xxdu3aRVVVFXPmzGlWojw+\nPp4tW7aQmppKXV0do0aNYuPGjc4Z3QiVKFe6NRWff07BqtVUf/klfadNY+ijjxA6blyr5+QdKaVP\n3yC+lP1EEMiokEGOpcyCJR4Y0SeIsUMjHLNBcR6VKHcWdRY9lNr8fAqf/Q1nt24l6LLhxDz3HBGz\nr2uXAN/JI6UMHzOAbae/ILG6moAx10KAc1nWe7NLmDJyAIEBKh6oKE6hzqKH4a6u5syGDZx++RVw\nu4lesoSou+8iICysXeefL6umrLCSsbOGcOTUEe6qOO+oyuzZqlq+OnWOOYnDHLNBURR1Fj0GYwzn\nduygcE0qtbm5RMyezdB/XU5wTEy7zne7DaeOnyXrE2vCUOXg09QXuEmqqXVMZRZgX04pxqh4oKI4\njTqLHkC1y8Wp1as5//920ychgZEbN9JvxvQ2z6s6X0tOVjHZmcXkHDxD1flaJEC4PHkwx4MtmfKk\nqInQ17lV6fZmlxAgMGWEM0q3iqJYqNzHJUz92bMUrF7NsQU/ovJgFkMfe4zRb21u0VEYYzidW87e\n/zrB5mf3smHZP9m+PoucrDPEJUZx3V2TuPPZK5lzXxKZpz4npraOqDFzmr2Wv9ibXcK4Yf2JCA12\n1I6ejkqU+x5vSJQXFxdzzTXXkJCQwOzZsz0S5f5AexaXIKa+ntK//Y2i3z1HfWkpAxbdzOClSwka\nePFQTW11PblflZCdeZrsA8WUl1QDED0inCuuH0VcYhRDRvUnoEnwOLMwg29VVzsar6h3GzJySrhp\naqxjNvQWGgT1Nm7cSFpaGi+88EKnrrN582YCAgIYP368N81rlQaJ8nR70a7uSoNE+R//+MeLjjWI\nNGZkZHhmozfHqlWruP7661m2bBlPPfUUqamprFq1ypdme1BncYlRkZ7OqadWUZWVRVjKFQx79FFC\nG6UNApQVVZJ94DTZmcWcPFxKfZ2b4D6BjJgwiG/PiyJuUhT9BrQ8e/V05Wnya8/yUxMCw7qujtlZ\nviw4y/maeo1XOMz777/PypUrqa6uJiEhgQ0bNtCvXz+WL1/Oe++9R1BQENdffz3z5s1j69atfPLJ\nJzz55JOdmsENlkT5mjVrMMYwf/58Vq9eDVgS5WvXruWyyy5jzJgxhIeH89xzzzUrUX7vvfd6ZL+3\nb9/Ovn37OHr0KLfffjvl5eUEBATwhz/8genTp7Njxw5WrVpFVFQU+/fvZ9GiRYwdO5bf//73VFdX\ns2XLFkaNGsWtt95KZGQkWVlZ5OTk8Oc//5n169ezZ88eZs2axfr16wFLojw9PZ3KykoWLVrk0cFq\nkFavr6/32NpAg62HDh1q9bN55513+PTTTwG47bbbmDNnjjoL5UJqT52i8DdrOfv3vxM0dCiXrf0N\n/efORUSor3OT7yrlxIFicg4UU1JQAcCAoX1JvCqGuKQoLhszgMDg9o06ZhbuB2Dy0KmOpsym96KV\n8f75xmFOf+1dldDoEeF89+axXbqGSpR3L4ny4uJij1xITEwM+fn5nbpOZ1Bn0c1x19RwZuOrnH7p\nJairI+oXPyf6nnuorA3ky935VnD60Blqq+oJCBJiEgYw6buWgxgwpG+n6sw8vp1AY5gw9kYvt6Zj\n7M0uYUhEH2IHti/tV/E+jSXKwVKPvfLKKy+QKL/hhhsumE3dFRpLlAMeifKqqiqPRDnAwoULycnJ\nASyJ8uTkZKB5ifIdO3YAlkT5/fffz/79+wkKCuLo0aOeehskyoGLJMp3797tKdecRDngkShPTExk\n06ZNrF+/nrq6OvLy8sjKyvKUa5Ao98Z6FkC75k15C586CxGZAzwPBALrjDHPNFPmZuBJwAD7jTE/\nsfePBNYBI+xjc40xJ3xpb3fCGEP5h7s49cwz1ObkEH7ttcitv+R4USAfPX+QwmxL2bNfZAgJKUOJ\nS4widvxAQkK7fku/KEhjbE0toQmzu3ytrpCWXcIVcQP9+oNwiq72AHyFUYnybiVRHhUVRVFREYMH\nD+bkyZMMG+a/+Uc+cxYiEgi8CPwQyAU+F5EtxpisRmUSgIeBWcaYEhEZ0ugSfwFWGWO2i0g44PaV\nrd2N6mPHOfX005TuTuPc+O9x7s6HySsKpHLj14jA0NGRTF8QT1xiFNGx4V79Z+o2bg5WFTI3KNLR\nlNlTZ6vILank9pmjHLNBUYnyjuJrifL58+fz6quvsmzZMl599VUWLFjgTfNbxZc9i2mAyxhzDEBE\nXgMWAFmNytwDvGiMKQEwxhTaZScCQcaY7fZ+7w7mdlPqzp3j+HN/5uh/H6U4ehpl312EMUKfggBG\nThpEXKIVnA4N910a6YmCDMrFkDQk2Wd1tIe9vShe0Z1RifKO4WuJ8kceeYSbb76ZP/3pT4wePZrX\nX3/d621oCfGFdwUQkYXAHGPM3fb2z4Dpxpj7G5V5GzgMzMIaqnrSGPNfIvIj4G6gBhgN7AB+bYy5\neKVzm5SUFJOWltZhO13709j1u686fJ4vMBJKTR/rn2N5n5MUDDxIYeRBSvtlY8Q/HauqACgKElKK\nf0YubU/s8xUlFTWcq6oj88nZhAT1zOlAhw4dukAAT2md8vJywsPDqa2tZcGCBfziF7/wxBDmz5/P\nc889R3x8vKccWKmmZ86cYe3atU6aDsCzzz7LkCFDLlg32980950Tkb3GmJS2zvVlz6K5sZGmnikI\nSACuBmKBf4pIor3/u0AykAO8DtwOrL+gApF7gXsBRo4c2Skjg/qEIqagU+d6n3pKBmTzdewRakOs\nyTZ9gKGEXfzJ+c4EJtX1p37w90gIcDb/4Yq4QT3WUSgdRyXKncWXPYvvYPUUZtvbDwMYY55uVOYl\n4FNjzEZ7eyfwa6xexjPGmKvt/T8DZhhjlrRUX2d7ForiFNqzUPxNV3oWvnxs+xxIEJHRIhICLAa2\nNCnzNvB9ABGJBsYCx+xzB4pIw+DeD7gw1qEoiqL4EZ85C2NMHXA/sA04BLxhjDkoIitFZL5dbBtQ\nLCJZwIfAcmNMsR2bWAbsFJFMrCGtV3xlq6I4ha969orSlK5+13w2DOVvdBhKudQ4fvw4ERERREVF\n9Yq5JIpzGGMoLi7m3LlzjB49+oJj3SHArShKK8TGxpKbm0tRUZHTpii9gNDQUGJjOy/Kqc5CURwi\nODj4oqc8RemuaF6ioiiK0ibqLBRFUZQ2UWehKIqitEmPyYYSkSIguwuXiAZOe8kcJ+kp7QBtS3el\np7Slp7QDutaWOGNM84JVjegxzqKriEhae9LHujs9pR2gbemu9JS29JR2gH/aosNQiqIoSpuos1AU\nRVHaRJ3FN7zstAFeoqe0A7Qt3ZWe0pae0g7wQ1s0ZqEoiqK0ifYsFEVRlDbptc5CRJ4VkS9F5AsR\neUtEBrRQbo6IfCUiLhH5tb/tbAsR+Z8iclBE3CLSYjaEiJwQkUwR2Sci3VJxsQNt6db3BEBEBonI\ndhE5Yv9tdn1YEam378k+EWkq4e8YbX3GItJHRF63j+8RkVH+t7J9tKMtt4tIUaP7cLcTdraFiGwQ\nkUIROdDCcRGR/7Db+YWITPWqAcaYXvkCrsNa5xtgDbCmmTKBwFEgHggB9gMTnba9iY0TgHHALiCl\nlXIngGin7e1qWy6Fe2LbmYq1FDBYC3pd9P2yj5U7bWtnPmPgfwEv2e8XA687bXcX2nI78ILTtraj\nLVcBU4EDLRyfC7yPtaTDDGCPN+vvtT0LY8wHxlpzA+BTrGVdmzINcBljjhljaoDXgAX+srE9GGMO\nGWO6xyLiXaSdben298RmAfCq/f5V4EcO2tJR2vMZN27fm8A10j111i+V70ubGGM+Bs60UmQB8Bdj\n8SkwQESGe6v+XussmnAnlkduSgzwdaPtXHvfpYgBPhCRvfba5Zcql8o9GWqMyQew/w5poVyoiKSJ\nyKci0l0cSns+Y08Z+6GrDIjyi3Udo73flx/bQzdvisgI/5jmdXz62+jREuUisgMY1syhR40x79hl\nHgXqgP/b3CWa2ef39LH2tKMdzDLG5InIEGC7iHxpP6n4FS+0pVvcE2i9LR24zEj7vsQD/xCRTGPM\nUe9Y2Gna8xl3m/vQBu2x8+/AJmNMtYj8HKvH9AOfW+Z9fHpPerSzMMZc29pxEbkNmAdcY+xBvybk\nAo2fMmKBPO9Z2D7aakc7r5Fn/y0Ukbewuud+dxZeaEu3uCfQeltE5JSIDDfG5NtDAYUtXKPhvhwT\nkV1AMtYYu5O05zNuKJMrIkFAJK0PkThFm20xxhQ32nwFK4Z5KeLT30avHYYSkTnAQ8B8Y0xFC8U+\nBxJEZLSIhGAF8rpNxkp7EZF+IhLR8B4ruN9sRsUlwKVyT7YAt9nvbwMu6jWJyEAR6WO/jwZmAVl+\ns7Bl2vMZN27fQuAfLTxwOU2bbWkyrj8fOORH+7zJFuBf7KyoGUBZw1CoV3A6wu/UC3Bhje/ts18N\nmR2XAVsblZsLHMZ62nvUabubacf/wHqiqAZOAduatgMrE2S//TrYHdvR3rZcCvfEtjEK2Akcsf8O\nsvenAOvs9zOBTPu+ZAJ3OW13a58xsBLr4QogFPir/Tv6DIh32uYutOVp+3exH/gQGO+0zS20YxOQ\nD9Tav5O7gJ8DP7ePC/Ci3c5MWsmO7MxLZ3AriqIobdJrh6EURVGU9qPOQlEURWkTdRaKoihKm6iz\nUBRFUdpEnYWiKIrSJuosFKUDiEh5F89/056tjYiEi8ifROSorbb7sYhMF5EQ+32PnjSrXFqos1AU\nPyEik4BAY8wxe9c6rFnPCcaYSVjqp9HGErzbCSxyxFBFaQZ1ForSCexZss+KyAF7nZBF9v4AEfmD\n3VN4V0S2ishC+7SfYs/kFpHLgenAY8YYN1iSH8aY9+yyb9vlFaVboN1cRekcNwFTgG8B0cDnIvIx\nlmTHKCAJS2n2ELDBPmcW1ixcgEnAPmNMfQvXPwB82yeWK0on0J6FonSOK7GUSuuNMaeAj7D+uV8J\n/NUY4zbGFGDJRzQwHChqz8VtJ1LToOmlKE6jzkJROkdLC/20tgBQJZamElhaRN8SkdZ+g32Aqk7Y\npiheR52FonSO73kr9AAAAP9JREFUj4FFIhIoIoOxlrz8DPhvrIV0AkRkKHB1o3MOAWMAjLVmRRqw\nomGFORFJEJEF9vsooMgYU+uvBilKa6izUJTO8RbwBZZS6T+Af7WHnf6GpQh6APgTsAdrFTmA97jQ\nedyNtXiSS0QysdZSaFh/4PvAVt82QVHaj6rOKoqXEZFwY0y53Tv4DGuVwgIRCcOKYcxqJbDdcI3N\nwMOmh6yvrlz6aDaUonifd0VkABAC/Lvd48AYUykiT2Cti5zT0sn2Ij1vq6NQuhPas1AURVHaRGMW\niqIoSpuos1AURVHaRJ2FoiiK0ibqLBRFUZQ2UWehKIqitIk6C0VRFKVN/j/1YwRIV19OkAAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14c0f320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "accuracy_s1 =np.array(accuracy_s).reshape(len(C_s),len(gamma_s))\n",
    "x_axis = np.log10(C_s)\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    plt.plot(x_axis, np.array(accuracy_s1[:,j]), label = ' Test - log(gamma)' + str(np.log10(gamma)))\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuracy' )\n",
    "plt.savefig('RBF_SVM_Otto.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "accuracy: 0.7973856209150327 C=0.215443 gamma=0.100000 这个时候结果比较理想一点"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "现在将分割数据集改为随机，测试结果可能会发生差异"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
